├── .colcon ├── build │ └── .gitkeep ├── install │ └── .gitkeep └── log │ └── .gitkeep ├── .docker ├── Dockerfile └── entrypoint.sh ├── .github └── workflows │ ├── lint.yaml │ └── ros2.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── LICENSE ├── README.md ├── dependencies.repos ├── docker-compose.yml ├── docs ├── COLCON_IGNORE ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── conf.py │ ├── demos │ ├── docs │ │ └── panda_collecting_toycars.rst │ └── index.rst │ └── index.rst ├── grab2_behavior_tree ├── CMakeLists.txt ├── behavior_trees │ ├── bt_groot2.btproj │ ├── collect_cubes.xml │ ├── collect_toys.xml │ ├── lookaround_demo.xml │ ├── stack_cubes.xml │ └── target_ik_demo.xml ├── include │ └── grab2_behavior_tree │ │ ├── detect_object.hpp │ │ ├── get_grasp.hpp │ │ ├── get_grasp_hardcoded.hpp │ │ ├── get_trajectory_hardcoded.hpp │ │ ├── grip.hpp │ │ ├── move.hpp │ │ ├── plan.hpp │ │ └── utils.hpp ├── launch │ ├── bt_demo.launch.py │ └── grab2_engine.launch.py ├── package.xml ├── plugins │ ├── detect_object.cpp │ ├── get_grasp.cpp │ ├── get_grasp_hardcoded.cpp │ ├── get_trajectory_hardcoded.cpp │ ├── grip.cpp │ ├── move.cpp │ └── plan.cpp └── src │ └── main.cpp ├── grab2_controller ├── CMakeLists.txt ├── config │ ├── panda │ │ ├── initial_positions.yaml │ │ └── ros2_controllers.yaml │ ├── rviz2 │ │ ├── 3d_detection_cfg.yaml │ │ └── rviz2_config.rviz │ └── ur │ │ ├── initial_positions.yaml │ │ └── ros2_controllers.yaml ├── description │ ├── panda │ │ ├── panda.ros2_control.xacro │ │ └── panda.urdf.xacro │ ├── robotiq_2f_140 │ │ ├── README.md │ │ ├── meshes │ │ │ ├── collision │ │ │ │ ├── robotiq_arg2f_140_inner_finger.stl │ │ │ │ ├── robotiq_arg2f_140_inner_knuckle.stl │ │ │ │ ├── robotiq_arg2f_140_outer_finger.stl │ │ │ │ ├── robotiq_arg2f_140_outer_knuckle.stl │ │ │ │ ├── robotiq_arg2f_base_link.stl │ │ │ │ └── robotiq_arg2f_coupling.stl │ │ │ └── visual │ │ │ │ ├── robotiq_arg2f_140_inner_finger.stl │ │ │ │ ├── robotiq_arg2f_140_inner_knuckle.stl │ │ │ │ ├── robotiq_arg2f_140_outer_finger.stl │ │ │ │ ├── robotiq_arg2f_140_outer_knuckle.stl │ │ │ │ ├── robotiq_arg2f_base_link.stl │ │ │ │ └── robotiq_arg2f_coupling.stl │ │ └── robotiq_2f_140.xacro │ └── ur │ │ ├── ur.ros2_control.xacro │ │ └── ur.urdf.xacro ├── launch │ ├── franka_controllers.launch.py │ ├── ur10e_suction_gripper_controllers.launch.py │ └── ur5e_robotiq_2f_140_controllers.launch.py └── package.xml ├── grab2_curobo_planner ├── config │ ├── table.yaml │ └── toybox.yaml ├── grab2_curobo_planner │ ├── __init__.py │ ├── curobo │ │ ├── __init__.py │ │ └── motion_generation.py │ └── planning_server.py ├── launch │ └── planning_server.launch.py ├── package.xml ├── resource │ └── grab2_curobo_planner ├── setup.cfg ├── setup.py └── test │ ├── test_copyright.py │ ├── test_flake8.py │ └── test_pep257.py ├── grab2_grasp_generator ├── CMakeLists.txt ├── config │ └── isaac_grasp.yaml ├── include │ └── grab2_grasp_generator │ │ ├── grasp_generator.hpp │ │ └── visibility_control.h ├── package.xml ├── src │ ├── grasp_generator.cpp │ └── main.cpp └── test │ ├── CMakeLists.txt │ └── test_grasp_generator.cpp ├── grab2_msgs ├── CMakeLists.txt ├── action │ ├── ComputePlanThroughPoses.action │ ├── ComputePlanToPose.action │ └── GetIsaacGrasp.action └── package.xml ├── grab2_planner ├── CMakeLists.txt ├── config │ └── panda_move_group.rviz ├── include │ └── grab2_planner │ │ └── planner_server.hpp ├── launch │ └── planning_server.launch.py ├── package.xml └── src │ ├── main.cpp │ └── planner_server.cpp ├── grab2_ros_common ├── CMakeLists.txt ├── include │ └── grab2_ros_common │ │ └── node.hpp └── package.xml └── isaac_sim_worlds ├── assets ├── objects │ ├── plate.usd │ └── toy_car.usd ├── robots │ ├── suction_gripper.usd │ └── ur10e_short_suction.usd └── worlds │ └── toybox_world │ ├── Textures │ ├── Mat_Box_baseColor.jpg │ ├── Mat_Box_metallicRoughness_metal_scale0.jpg │ ├── Mat_Box_metallicRoughness_rough.jpg │ ├── Mat_Box_normal.jpg │ ├── Mat_Light_baseColor.png │ ├── Mat_Light_emissive.jpg │ ├── Mat_Light_metallicRoughness_metal_scale0.jpg │ ├── Mat_Light_metallicRoughness_rough.jpg │ ├── Mat_Room_baseColor.jpg │ ├── Mat_Room_metallicRoughness_metal.jpg │ ├── Mat_Room_metallicRoughness_rough.jpg │ ├── Mat_Room_normal.jpg │ ├── Mat_Rug_baseColor_scale1.jpg │ ├── Mat_Rug_metallicRoughness_metal_scale0.jpg │ ├── Mat_Rug_metallicRoughness_rough.jpg │ ├── Mat_Rug_normal.jpg │ ├── Mat_Toys_baseColor.png │ ├── Mat_Toys_emissive.jpg │ ├── Mat_Toys_metallicRoughness_metal.jpg │ ├── Mat_Toys_metallicRoughness_rough.jpg │ └── Mat_Toys_normal.jpg │ └── world.usd ├── isaac_py.sh ├── panda_bin_cubes.py ├── panda_toybox.py └── ur10e_stack_plates.py /.colcon/build/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elsayedelsheikh/grab2/1a308a68997cd4eac40fb946b0c465f2a709189d/.colcon/build/.gitkeep -------------------------------------------------------------------------------- /.colcon/install/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elsayedelsheikh/grab2/1a308a68997cd4eac40fb946b0c465f2a709189d/.colcon/install/.gitkeep -------------------------------------------------------------------------------- /.colcon/log/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elsayedelsheikh/grab2/1a308a68997cd4eac40fb946b0c465f2a709189d/.colcon/log/.gitkeep -------------------------------------------------------------------------------- /.docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvcr.io/nvidia/pytorch:23.08-py3 AS base 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | 5 | # Install language 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | locales \ 8 | && locale-gen en_US.UTF-8 \ 9 | && update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 \ 10 | && rm -rf /var/lib/apt/lists/* 11 | ENV LANG=en_US.UTF-8 12 | 13 | # Install timezone 14 | RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime \ 15 | && export DEBIAN_FRONTEND=noninteractive \ 16 | && apt-get update \ 17 | && apt-get install -y --no-install-recommends tzdata \ 18 | && dpkg-reconfigure --frontend noninteractive tzdata \ 19 | && rm -rf /var/lib/apt/lists/* 20 | 21 | RUN apt-get update && apt-get -y upgrade \ 22 | && rm -rf /var/lib/apt/lists/* 23 | 24 | # Install ROS2 25 | RUN apt-get update && apt-get install -y --no-install-recommends \ 26 | curl \ 27 | software-properties-common \ 28 | && add-apt-repository universe \ 29 | && export ROS_APT_SOURCE_VERSION=$(curl -s https://api.github.com/repos/ros-infrastructure/ros-apt-source/releases/latest | grep -F "tag_name" | awk -F\" '{print $4}') \ 30 | && curl -L -o /tmp/ros2-apt-source.deb "https://github.com/ros-infrastructure/ros-apt-source/releases/download/${ROS_APT_SOURCE_VERSION}/ros2-apt-source_${ROS_APT_SOURCE_VERSION}.$(. /etc/os-release && echo $VERSION_CODENAME)_all.deb" \ 31 | && dpkg -i /tmp/ros2-apt-source.deb \ 32 | && apt-get update && apt-get install -y --no-install-recommends \ 33 | python3-argcomplete \ 34 | ros-humble-desktop \ 35 | && rm -rf /var/lib/apt/lists/* 36 | 37 | # Deal with getting tons of debconf messages 38 | # See: https://github.com/phusion/baseimage-docker/issues/58 39 | RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections 40 | 41 | # add GL: 42 | RUN apt-get update && apt-get install -y --no-install-recommends \ 43 | pkg-config \ 44 | libglvnd-dev \ 45 | libgl1-mesa-dev \ 46 | libegl1-mesa-dev \ 47 | libgles2-mesa-dev && \ 48 | rm -rf /var/lib/apt/lists/* 49 | 50 | # Env vars for the nvidia-container-runtime. 51 | ENV NVIDIA_VISIBLE_DEVICES all 52 | ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute 53 | ENV QT_X11_NO_MITSHM=1 54 | 55 | ENV ROS_DISTRO=humble 56 | ENV AMENT_PREFIX_PATH=/opt/ros/humble 57 | ENV COLCON_PREFIX_PATH=/opt/ros/humble 58 | ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/ros/humble/lib/x86_64-linux-gnu:/opt/ros/humble/lib" 59 | ENV PATH=/opt/ros/humble/bin:$PATH 60 | ENV PYTHONPATH=/opt/ros/humble/local/lib/python3.10/dist-packages:/opt/ros/humble/lib/python3.10/site-packages 61 | ENV ROS_PYTHON_VERSION=3 62 | ENV ROS_VERSION=2 63 | ENV ROS_AUTOMATIC_DISCOVERY_RANGE=SUBNET 64 | ENV DEBIAN_FRONTEND= 65 | 66 | RUN apt-get update && apt-get install -y \ 67 | tzdata \ 68 | software-properties-common \ 69 | && dpkg-reconfigure -f noninteractive tzdata \ 70 | && add-apt-repository -y ppa:git-core/ppa \ 71 | && apt-get update && apt-get install -y \ 72 | curl \ 73 | lsb-core \ 74 | wget \ 75 | build-essential \ 76 | cmake \ 77 | git \ 78 | git-lfs \ 79 | iputils-ping \ 80 | make \ 81 | openssh-server \ 82 | openssh-client \ 83 | libeigen3-dev \ 84 | libssl-dev \ 85 | python3-pip \ 86 | python3-ipdb \ 87 | python3-tk \ 88 | python3-wstool \ 89 | sudo git bash unattended-upgrades \ 90 | apt-utils \ 91 | terminator \ 92 | glmark2 \ 93 | && rm -rf /var/lib/apt/lists/* 94 | 95 | # push defaults to bashrc: 96 | RUN apt-get update && apt-get install --reinstall -y \ 97 | libmpich-dev \ 98 | hwloc-nox libmpich12 mpich \ 99 | && rm -rf /var/lib/apt/lists/* 100 | 101 | # This is required to enable mpi lib access: 102 | ENV PATH="${PATH}:/opt/hpcx/ompi/bin" 103 | ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/hpcx/ompi/lib" 104 | 105 | ENV TORCH_CUDA_ARCH_LIST "7.0+PTX" 106 | ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" 107 | 108 | # Get CuRobo 109 | # COPY curobo /pkgs/curobo # If you have local curobo package, you can copy it directly 110 | RUN mkdir /pkgs && cd /pkgs && git clone https://github.com/NVlabs/curobo.git 111 | RUN cd /pkgs/curobo && pip3 install .[dev,usd] --no-build-isolation 112 | 113 | # Setup ROS2 Workspace 114 | RUN mkdir -p /ros2_ws/src 115 | WORKDIR /ros2_ws 116 | COPY dependencies.repos ./src 117 | 118 | # Install ROS2 dependencies 119 | RUN apt-get update && apt-get install -y --no-install-recommends \ 120 | python3-vcstool \ 121 | ros-${ROS_DISTRO}-rviz2 \ 122 | ros-${ROS_DISTRO}-xacro \ 123 | ros-${ROS_DISTRO}-ros2-control \ 124 | ros-${ROS_DISTRO}-behaviortree-cpp \ 125 | ros-${ROS_DISTRO}-tf-transformations \ 126 | ros-${ROS_DISTRO}-rmw-cyclonedds-cpp \ 127 | ros-${ROS_DISTRO}-hardware-interface \ 128 | ros-${ROS_DISTRO}-controller-manager \ 129 | ros-${ROS_DISTRO}-gripper-controllers \ 130 | ros-${ROS_DISTRO}-position-controllers \ 131 | ros-${ROS_DISTRO}-joint-state-broadcaster \ 132 | ros-${ROS_DISTRO}-topic-based-ros2-control \ 133 | && rm -rf /var/lib/apt/lists/* 134 | 135 | RUN vcs import src < src/dependencies.repos 136 | 137 | ########################################### 138 | # Develop image 139 | ########################################### 140 | FROM base AS dev 141 | 142 | ENV DEBIAN_FRONTEND=noninteractive 143 | RUN apt-get update && apt-get install -y --no-install-recommends \ 144 | bash-completion \ 145 | build-essential \ 146 | cmake \ 147 | gdb \ 148 | git \ 149 | openssh-client \ 150 | python3-argcomplete \ 151 | python3-pip \ 152 | ros-dev-tools \ 153 | ros-${ROS_DISTRO}-ament-* \ 154 | vim \ 155 | && rm -rf /var/lib/apt/lists/* 156 | 157 | RUN rosdep init || echo "rosdep already initialized" 158 | 159 | ARG USERNAME=ros 160 | ARG USER_UID=1000 161 | ARG USER_GID=$USER_UID 162 | 163 | # Check if "ubuntu" user exists, delete it if it does, then create the desired user 164 | RUN if getent passwd ubuntu > /dev/null 2>&1; then \ 165 | userdel -r ubuntu && \ 166 | echo "Deleted existing ubuntu user"; \ 167 | fi && \ 168 | groupadd --gid $USER_GID $USERNAME && \ 169 | useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME && \ 170 | echo "Created new user $USERNAME" 171 | 172 | # Add sudo support for the non-root user 173 | RUN apt-get update && apt-get install -y sudo \ 174 | && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME\ 175 | && chmod 0440 /etc/sudoers.d/$USERNAME \ 176 | && rm -rf /var/lib/apt/lists/* 177 | 178 | # Set up autocompletion for user 179 | RUN apt-get update && apt-get install -y --no-install-recommends git-core bash-completion \ 180 | && echo "if [ -f /opt/ros/${ROS_DISTRO}/setup.bash ]; then source /opt/ros/${ROS_DISTRO}/setup.bash; fi" >> /home/$USERNAME/.bashrc \ 181 | && echo "if [ -f /usr/share/colcon_argcomplete/hook/colcon-argcomplete.bash ]; then source /usr/share/colcon_argcomplete/hook/colcon-argcomplete.bash; fi" >> /home/$USERNAME/.bashrc \ 182 | && rm -rf /var/lib/apt/lists/* 183 | 184 | ENV DEBIAN_FRONTEND= 185 | ENV AMENT_CPPCHECK_ALLOW_SLOW_VERSIONS=1 186 | -------------------------------------------------------------------------------- /.docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Re-install MPICH - Required for CuRobo 5 | sudo apt-get update 6 | sudo apt-get -y install --reinstall -y \ 7 | libmpich-dev hwloc-nox libmpich12 mpich 8 | 9 | # Install ROS2 dependencies 10 | rosdep update --rosdistro=${ROS_DISTRO} 11 | rosdep install --from-paths src --ignore-src -r -y --rosdistro=${ROS_DISTRO} 12 | 13 | # Build 14 | colcon build --symlink-install 15 | 16 | # Source workspace 17 | if [ -f /ros2_ws/install/setup.bash ] 18 | then 19 | source /ros2_ws/install/setup.bash 20 | fi 21 | 22 | # Execute the command passed into this entrypoint 23 | exec "$@" 24 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | pre-commit: 8 | name: pre-commit 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v5 12 | - uses: actions/setup-python@v5 13 | - uses: pre-commit/action@v3.0.1 14 | 15 | action-ros-lint: 16 | name: ament_${{ matrix.linter }} 17 | runs-on: ubuntu-latest 18 | container: 19 | image: rostooling/setup-ros-docker:ubuntu-noble-ros-rolling-ros-base-latest 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | linter: 24 | [uncrustify, cpplint, lint_cmake, xmllint, flake8, pep257] 25 | steps: 26 | - uses: actions/checkout@v5 27 | - uses: ros-tooling/action-ros-lint@0.1.4 28 | with: 29 | linter: ${{ matrix.linter }} 30 | package-name: "*" 31 | -------------------------------------------------------------------------------- /.github/workflows/ros2.yaml: -------------------------------------------------------------------------------- 1 | name: ROS2 CI 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened, ready_for_review] 6 | 7 | push: 8 | branches: [main] 9 | 10 | jobs: 11 | build-and-test: 12 | name: build-and-test-against-${{ matrix.ros-distro }} 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | include: 18 | - os: [ubuntu-24.04] 19 | ros-distro: "kilted" 20 | - os: [ubuntu-24.04] 21 | ros-distro: "jazzy" 22 | - os: [ubuntu-22.04] 23 | ros-distro: "humble" 24 | steps: 25 | - name: Install yaml-cpp deps 26 | run: sudo apt-get install -y libyaml-cpp-dev 27 | 28 | - name: Checkout 29 | uses: actions/checkout@v5 30 | 31 | - name: Setup ROS 2 32 | uses: ros-tooling/setup-ros@0.7.15 33 | with: 34 | required-ros-distributions: ${{ matrix.ros-distro }} 35 | 36 | - name: Build and Test 37 | uses: ros-tooling/action-ros-ci@0.4.5 38 | with: 39 | target-ros2-distro: ${{ matrix.ros-distro }} 40 | vcs-repo-file-url: ${GITHUB_WORKSPACE}/dependencies.repos 41 | colcon-defaults: | 42 | { 43 | "test": { 44 | "parallel-workers" : 1 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/__pycache__/ 2 | .hypothesis/* 3 | .thumbs/ 4 | 5 | .colcon/ 6 | BehaviorTree.ROS2/ 7 | 8 | # Sphinx documentation 9 | docs/build/ 10 | docs/venv/ 11 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # To use: 2 | # 3 | # pre-commit run -a 4 | # 5 | # Or: 6 | # 7 | # pre-commit install # (runs every time you commit in git) 8 | # 9 | # To update this file: 10 | # 11 | # pre-commit autoupdate 12 | # 13 | # To Install pre-commit 14 | # 15 | # pip install pre-commit --break-system-packages 16 | # 17 | # See https://github.com/pre-commit/pre-commit 18 | repos: 19 | # Standard hooks 20 | - repo: https://github.com/pre-commit/pre-commit-hooks 21 | rev: v6.0.0 22 | hooks: 23 | - id: check-added-large-files 24 | - id: check-ast 25 | - id: check-builtin-literals 26 | - id: check-case-conflict 27 | - id: check-docstring-first 28 | - id: check-executables-have-shebangs 29 | - id: check-json 30 | - id: check-merge-conflict 31 | - id: check-symlinks 32 | - id: check-toml 33 | - id: check-vcs-permalinks 34 | - id: check-yaml 35 | - id: debug-statements 36 | - id: destroyed-symlinks 37 | - id: detect-private-key 38 | - id: end-of-file-fixer 39 | - id: fix-byte-order-marker 40 | - id: forbid-new-submodules 41 | - id: mixed-line-ending 42 | - id: name-tests-test 43 | - id: requirements-txt-fixer 44 | - id: sort-simple-yaml 45 | - id: trailing-whitespace 46 | 47 | # Autoformats Python code. 48 | - repo: https://github.com/psf/black.git 49 | rev: 25.1.0 50 | hooks: 51 | - id: black 52 | args: ["--skip-string-normalization"] 53 | 54 | # Finds spelling issues in code. 55 | - repo: https://github.com/codespell-project/codespell 56 | rev: v2.4.1 57 | hooks: 58 | - id: codespell 59 | args: ["--write-changes", "-L", "reacher"] # Provide a comma-separated list of misspelled words that codespell should ignore (for example: '-L', 'word1,word2,word3'). 60 | exclude: \.(svg|pyc|stl|dae|lock)$ 61 | 62 | - repo: https://github.com/pre-commit/mirrors-clang-format 63 | rev: v21.1.0 64 | hooks: 65 | - id: clang-format 66 | files: \.(c|cc|cxx|cpp|frag|glsl|h|hpp|hxx|ih|ispc|ipp|java|m|proto|vert)$ 67 | # -i arg is included by default by the hook 68 | args: ["-fallback-style=none"] 69 | 70 | - repo: https://github.com/adrienverge/yamllint 71 | rev: v1.37.1 72 | hooks: 73 | - id: yamllint 74 | args: 75 | [ 76 | "--no-warnings", 77 | "--config-data", 78 | "{extends: default, rules: {line-length: disable, braces: {max-spaces-inside: 1}, indentation: disable}}", 79 | ] 80 | types: [text] 81 | files: \.(yml|yaml)$ 82 | 83 | # Checks links in markdown files. 84 | - repo: https://github.com/tcort/markdown-link-check 85 | rev: v3.13.7 86 | hooks: 87 | - id: markdown-link-check 88 | 89 | # CPP formatting 90 | - repo: https://github.com/pre-commit/mirrors-clang-format 91 | rev: v21.1.0 92 | hooks: 93 | - id: clang-format 94 | types_or: [c++, c] 95 | args: ['-fallback-style=none', '-i'] 96 | 97 | exclude: '(.colcon)/.*' 98 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version, and other tools you might need 8 | build: 9 | os: ubuntu-24.04 10 | tools: 11 | python: "3.12" 12 | 13 | # Build documentation in the "docs/" directory with Sphinx 14 | sphinx: 15 | configuration: docs/source/conf.py 16 | 17 | # Optionally build your docs in additional formats such as PDF and ePub 18 | formats: 19 | - pdf 20 | - epub 21 | 22 | # Optionally, but recommended, 23 | # declare the Python requirements required to build your documentation 24 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 25 | python: 26 | install: 27 | - requirements: docs/requirements.txt 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 ElSayed ElSheikh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grab2 Behavior Demos 2 | 3 | [![ROS2 CI](https://github.com/elsayedelsheikh/grab2/actions/workflows/ros2.yaml/badge.svg)](https://github.com/elsayedelsheikh/grab2/actions/workflows/ros2.yaml) 4 | 5 | This repository provides manipulation demos 6 | 7 | - Using BehaviorTree.CPP, Moveit2, Nvidia cuRobo, and ROS2 with Nvidia Isaac sim. 8 | - You can use this package with any simulator that supports topic based ROS2 control. 9 | - Docker containers are available as well. 10 | 11 | Available demos: 12 | 13 | - **Isaac Sim:** 14 | Check [Start Isaac Sim world](#2-start-the-isaac-sim-simulation-world) 15 | - **No Isaac Sim:** 16 | Run [mock hardware demo](#3-start-the-behavior-demo) 17 | 18 | Screenshot from 2025-07-27 16-36-30 19 | 20 | See the sections below for detailed instructions on each option. 21 | 22 | ## Project Overview 23 | 24 | This project demonstrates robot manipulation using a modular tech stack: 25 | 26 | - **BehaviorTree.CPP v4.6** for task-level logic and decision making 27 | - **Nvidia CuRobo** for fast, efficient, real-time trajectory planning 28 | - **ROS2** with **ros2_control** and `joint_trajectory_controller` for motion execution 29 | 30 | The behavior tree manages the decision-making flow, Nvidia CuRobo handles motion planning, and ROS2 `ros2_control` executes the robot's motion. 31 | 32 | Screenshot from 2025-07-27 16-29-25 33 | 34 | ### Demo Scenarios 35 | 36 | - **Simple Room:** 37 | Franka robot, a bin, and three cubes on a table 38 | - **Toybox:** 39 | Panda robot with car toys scattered on the ground 40 | 41 | ### Available Behavior Demos 42 | 43 | - `collect_cubes` 44 | - `collect_toys` 45 | - `lookaround_demo` 46 | - `target_ik_demo` 47 | 48 | For more information, see `grab2_behavior_tree/behavior_trees`. 49 | 50 | ### Available Planners 51 | 52 | - Moveit2 53 | - Nvidia CuRobo 54 | - Did you write one? Send a pull request to add it to this list. 55 | 56 | ### Available BehaviorTree Plugins 57 | 58 | - See `grab2_behavior_tree/plugins`. 59 | - Did you write one? Send a pull request to add it to this list. 60 | 61 | ## Requirements 62 | 63 | - NVIDIA GPU with driver version 550+ is required for simulation and demo containers. 64 | - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/) must be installed. 65 | - Ensure [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) is set up for GPU access in containers. 66 | 67 | ## Running Demos with Docker Compose 68 | 69 | This project uses Docker Compose to manage simulation and demo containers 70 | defined in `docker-compose.yml`. 71 | 72 | --- 73 | 74 | ### 1. Build the Containers 75 | 76 | Build all required images: 77 | 78 | ```sh 79 | docker compose build 80 | ``` 81 | 82 | --- 83 | 84 | ### 2. Start the Isaac Sim Simulation World 85 | 86 | You can start the Isaac Sim simulation world in two ways: 87 | 88 | - **Using Docker Compose:** 89 | Launch the simulation environment in a container with: 90 | 91 | ```sh 92 | docker compose up isaac-simple-room 93 | ``` 94 | 95 | or/ for the toybox world: 96 | 97 | ```sh 98 | docker compose up isaac-toybox 99 | ``` 100 | 101 | - **Using a Local Isaac Sim Installation (version 4.5):** 102 | If you have Isaac Sim installed locally, you can run the simulation script directly: 103 | 104 | ```sh 105 | cd isaac_sim_worlds 106 | ./isaac_py.sh panda_bin_cubes.py 107 | ``` 108 | 109 | or/ for the toybox world: 110 | 111 | ```sh 112 | cd isaac_sim_worlds 113 | ./isaac_py.sh panda_toybox.py 114 | ``` 115 | 116 | --- 117 | 118 | ### 3. Start the Behavior Demo 119 | 120 | In a separate terminal, start the behavior demo container: 121 | 122 | - Simple Room Demo (with Isaac Sim): 123 | 124 | ```sh 125 | docker compose up simple-room-demo 126 | ``` 127 | 128 | - Toybox Demo: 129 | 130 | ```sh 131 | docker compose up toybox-demo 132 | ``` 133 | 134 | - Simple Room Demo (no Isaac Sim, mock hardware): 135 | (No Isaac Sim required; can be run standalone, without objects to grab) 136 | 137 | ```sh 138 | docker compose up simple-room-demo-no-isaac 139 | ``` 140 | 141 | --- 142 | 143 | ### 4. Stopping the Demos 144 | 145 | To stop all running containers, press `Ctrl+C` in the terminal or run: 146 | 147 | ```sh 148 | docker compose down 149 | ``` 150 | -------------------------------------------------------------------------------- /dependencies.repos: -------------------------------------------------------------------------------- 1 | # List of repositories to use within your workspace 2 | # See https://github.com/dirk-thomas/vcstool 3 | repositories: 4 | BehaviorTree.ROS2: 5 | type: git 6 | url: https://github.com/elsayedelsheikh/BehaviorTree.ROS2.git 7 | version: deprecate_ament_target_deps 8 | topic_based_ros2_control: 9 | type: git 10 | url: https://github.com/PickNikRobotics/topic_based_ros2_control.git 11 | version: main 12 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | isaac-sim: 3 | container_name: isaacsim450 4 | image: nvcr.io/nvidia/isaac-sim:4.5.0 5 | environment: 6 | - ACCEPT_EULA=Y 7 | - PRIVACY_CONSENT=Y 8 | - ROS_DOMAIN_ID=0 9 | - RMW_IMPLEMENTATION=rmw_cyclonedds_cpp 10 | - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/isaac-sim/exts/isaacsim.ros2.bridge/humble/lib 11 | entrypoint: bash 12 | working_dir: /isaac_sim_worlds 13 | volumes: 14 | - ~/docker/isaac-sim/cache/kit:/isaac-sim/kit/cache:rw 15 | - ~/docker/isaac-sim/cache/ov:/root/.cache/ov:rw 16 | - ~/docker/isaac-sim/cache/pip:/root/.cache/pip:rw 17 | - ~/docker/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw 18 | - ~/docker/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw 19 | - ~/docker/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw 20 | - ~/docker/isaac-sim/data:/root/.local/share/ov/data:rw 21 | - ~/docker/isaac-sim/documents:/root/Documents:rw 22 | - ./isaac_sim_worlds:/isaac_sim_worlds:ro 23 | deploy: 24 | resources: 25 | reservations: 26 | devices: 27 | - driver: nvidia 28 | count: all 29 | capabilities: [gpu] 30 | network_mode: host 31 | 32 | isaac-simple-room: 33 | extends: isaac-sim 34 | command: ["./isaac_py.sh", "panda_bin_cubes.py"] 35 | 36 | isaac-toybox: 37 | extends: isaac-sim 38 | command: ["./isaac_py.sh", "panda_toybox.py"] 39 | 40 | grab2: 41 | container_name: grab2 42 | image: grab2 43 | build: 44 | context: . 45 | dockerfile: .docker/Dockerfile 46 | target: dev 47 | args: 48 | - USERNAME=${USERNAME:-ros} 49 | - USER_UID=${UID:-1000} 50 | - USER_GID=${UID:-1000} 51 | # Interactive Shell 52 | stdin_open: true 53 | tty: true 54 | # Networking 55 | network_mode: host 56 | ipc: host 57 | environment: 58 | - ROS_DOMAIN_ID=0 59 | - RMW_IMPLEMENTATION=rmw_cyclonedds_cpp 60 | # Allows graphical programs in the container. 61 | - DISPLAY=${DISPLAY} 62 | - QT_X11_NO_MITSHM=1 63 | - NVIDIA_DRIVER_CAPABILITIES=all 64 | working_dir: /ros2_ws 65 | user: ${USERNAME:-ros} 66 | volumes: 67 | - ./.docker/entrypoint.sh:/entrypoint.sh:ro 68 | - ./grab2_behavior_tree:/ros2_ws/src/grab2_behavior_tree:ro 69 | - ./grab2_controller:/ros2_ws/src/grab2_controller:ro 70 | - ./grab2_msgs:/ros2_ws/src/grab2_msgs:ro 71 | - ./grab2_curobo_planner:/ros2_ws/src/grab2_curobo_planner:ro 72 | - ./.colcon/build/:/ros2_ws/build/:rw 73 | - ./.colcon/install/:/ros2_ws/install/:rw 74 | - ./.colcon/log/:/ros2_ws/log/:rw 75 | # Allows graphical programs in the container. 76 | - /tmp/.X11-unix:/tmp/.X11-unix:rw 77 | - ${XAUTHORITY:-$HOME/.Xauthority}:/root/.Xauthority 78 | deploy: 79 | resources: 80 | reservations: 81 | devices: 82 | - driver: nvidia 83 | count: all 84 | capabilities: [gpu] 85 | entrypoint: ["/entrypoint.sh"] 86 | 87 | simple-room-demo: 88 | extends: grab2 89 | command: > 90 | ros2 launch grab2_behavior_tree bt_demo.launch.py 91 | robot:=panda 92 | world:=table 93 | behavior:=collect_cubes 94 | 95 | simple-room-demo-no-isaac: 96 | extends: grab2 97 | command: > 98 | ros2 launch grab2_behavior_tree bt_demo.launch.py 99 | robot:=panda 100 | world:=table 101 | behavior:=collect_cubes 102 | hardware:=mock_components 103 | 104 | toybox-demo: 105 | extends: grab2 106 | command: > 107 | ros2 launch grab2_behavior_tree bt_demo.launch.py 108 | robot:=panda 109 | world:=toybox 110 | behavior:=collect_toys 111 | -------------------------------------------------------------------------------- /docs/COLCON_IGNORE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elsayedelsheikh/grab2/1a308a68997cd4eac40fb946b0c465f2a709189d/docs/COLCON_IGNORE -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | esbonio 2 | sphinx 3 | sphinx-autobuild 4 | sphinx-copybutton 5 | sphinx_rtd_theme 6 | sphinxcontrib-youtube 7 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = 'grab2' 10 | copyright = '2025, ElSayed ElSheikh' 11 | author = 'ElSayed ElSheikh' 12 | release = '0.0.1' 13 | 14 | # -- General configuration --------------------------------------------------- 15 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 16 | 17 | extensions = [ 18 | "sphinx.ext.autodoc", 19 | "sphinx.ext.autosummary", 20 | "sphinx_rtd_theme", 21 | "sphinx_copybutton", 22 | "sphinxcontrib.youtube", 23 | ] 24 | 25 | templates_path = ['_templates'] 26 | exclude_patterns = [] 27 | 28 | 29 | # -- Options for HTML output ------------------------------------------------- 30 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 31 | 32 | html_theme = 'sphinx_rtd_theme' 33 | html_static_path = ['_static'] 34 | -------------------------------------------------------------------------------- /docs/source/demos/docs/panda_collecting_toycars.rst: -------------------------------------------------------------------------------- 1 | Collecting toys with Franka Panda 2 | ################################# 3 | 4 | In this demo, we will be using Franka Panda to clean up the playground after kids had missed up the place. 5 | 6 | So Franka will look for the toycars using the camera fixed at the end-effector, `lookaround_demo` behavior, whenever a toycar is detected, panda will try to grasp it, lift it up and throw it in the toybox and continues until there are no toys left on the ground. 7 | -------------------------------------------------------------------------------- /docs/source/demos/index.rst: -------------------------------------------------------------------------------- 1 | Demos 2 | ###### 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | docs/panda_collecting_toycars 8 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. grab2 documentation master file, created by 2 | sphinx-quickstart on Thu Sep 25 00:31:48 2025. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | grab2 documentation 7 | =================== 8 | 9 | grab2 brings together ROS 2, NVIDIA cuRobo, BehaviorTree.CPP and NVIDIA Isaac Sim; a practical setup for a typical manipulation pipeline from perception to motion planning and behavior orchestration. 10 | 11 | Setup 12 | ===== 13 | 14 | Supported Robots 15 | ---------------- 16 | 17 | +----------------------------------------------------------------+-------------+ 18 | | Robot | Status | 19 | +================================================================+=============+ 20 | | Franka panda with 2-finger girpper | Supported | 21 | +----------------------------------------------------------------+-------------+ 22 | | Universal Robot (UR5) with 2-finger gripper "`robotiq_2f_140`" | Supported | 23 | +----------------------------------------------------------------+-------------+ 24 | | Universal Robot (UR10) with suction gripper | Supported | 25 | +----------------------------------------------------------------+-------------+ 26 | | Tiago (Mobile manipulator) | In progress | 27 | +----------------------------------------------------------------+-------------+ 28 | 29 | Supported Motion Planners 30 | ------------------------- 31 | 32 | * Moveit2 33 | * NVIDIA cuRobo 34 | 35 | Supported Simulators 36 | ------------------------- 37 | 38 | * NVIDIA Isaac Sim >= 4.5 (any simulator that supports ROS2 topic based control) 39 | * Gazebo 40 | 41 | Grab2 Demos 42 | =========== 43 | 44 | .. toctree:: 45 | :maxdepth: 1 46 | 47 | demos/index 48 | -------------------------------------------------------------------------------- /grab2_behavior_tree/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.8) 2 | project(grab2_behavior_tree) 3 | 4 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 5 | add_compile_options(-Wall -Wextra -Wpedantic) 6 | endif() 7 | 8 | # find dependencies 9 | find_package(ament_cmake REQUIRED) 10 | find_package(rclcpp REQUIRED) 11 | find_package(rclcpp_action REQUIRED) 12 | find_package(rclcpp_components REQUIRED) 13 | find_package(ament_index_cpp REQUIRED) 14 | find_package(std_msgs REQUIRED) 15 | find_package(builtin_interfaces REQUIRED) 16 | find_package(geometry_msgs REQUIRED) 17 | find_package(tf2_geometry_msgs REQUIRED) 18 | find_package(vision_msgs REQUIRED) 19 | find_package(vision_msgs_rviz_plugins REQUIRED) 20 | find_package(control_msgs REQUIRED) 21 | find_package(trajectory_msgs REQUIRED) 22 | find_package(grab2_msgs REQUIRED) 23 | find_package(behaviortree_cpp REQUIRED) 24 | find_package(behaviortree_ros2 REQUIRED) 25 | 26 | set(CMAKE_CXX_STANDARD 17) 27 | 28 | set(THIS_PACK_DEPS 29 | rclcpp 30 | rclcpp_action 31 | rclcpp_components 32 | std_msgs 33 | builtin_interfaces 34 | geometry_msgs 35 | tf2_geometry_msgs 36 | vision_msgs 37 | trajectory_msgs 38 | ament_index_cpp 39 | control_msgs 40 | grab2_msgs 41 | behaviortree_cpp 42 | behaviortree_ros2 43 | ) 44 | 45 | # Libraries 46 | add_library(grab2_get_trajectory_hardcoded SHARED plugins/get_trajectory_hardcoded.cpp) 47 | add_library(grab2_get_grasp_hardcoded SHARED plugins/get_grasp_hardcoded.cpp) 48 | add_library(grab2_detect_object SHARED plugins/detect_object.cpp) 49 | add_library(grab2_get_grasp SHARED plugins/get_grasp.cpp) 50 | add_library(grab2_plan SHARED plugins/plan.cpp) 51 | add_library(grab2_move SHARED plugins/move.cpp) 52 | add_library(grab2_grip SHARED plugins/grip.cpp) 53 | 54 | list(APPEND plugin_libs 55 | grab2_get_trajectory_hardcoded 56 | grab2_get_grasp_hardcoded 57 | grab2_detect_object 58 | grab2_get_grasp 59 | grab2_plan 60 | grab2_move 61 | grab2_grip 62 | ) 63 | 64 | foreach(bt_plugin ${plugin_libs}) 65 | ament_target_dependencies(${bt_plugin} ${THIS_PACK_DEPS}) 66 | target_compile_definitions(${bt_plugin} PRIVATE BT_PLUGIN_EXPORT) 67 | target_include_directories(${bt_plugin} 68 | PRIVATE 69 | $ 70 | $) 71 | endforeach() 72 | 73 | # Executables 74 | add_executable(grab2_engine src/main.cpp) 75 | ament_target_dependencies(grab2_engine ${THIS_PACK_DEPS}) 76 | target_include_directories(grab2_engine PUBLIC 77 | $ 78 | $) 79 | 80 | # Install Libs & Exec 81 | install( 82 | TARGETS 83 | ${plugin_libs} 84 | grab2_engine 85 | ARCHIVE DESTINATION lib 86 | LIBRARY DESTINATION lib 87 | RUNTIME DESTINATION lib/${PROJECT_NAME} 88 | ) 89 | 90 | # Install directories 91 | install( 92 | DIRECTORY include/ 93 | DESTINATION include 94 | ) 95 | 96 | install( 97 | DIRECTORY behavior_trees launch 98 | DESTINATION share/${PROJECT_NAME} 99 | ) 100 | 101 | 102 | if(BUILD_TESTING) 103 | find_package(ament_lint_auto REQUIRED) 104 | 105 | set(ament_cmake_copyright_FOUND TRUE) 106 | ament_lint_auto_find_test_dependencies() 107 | endif() 108 | 109 | ament_package() 110 | -------------------------------------------------------------------------------- /grab2_behavior_tree/behavior_trees/bt_groot2.btproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /grab2_behavior_tree/behavior_trees/collect_cubes.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |