├── colcon_poetry_ros ├── task │ ├── __init__.py │ └── poetry │ │ ├── __init__.py │ │ ├── test.py │ │ └── build.py ├── dependencies │ ├── __init__.py │ └── install.py ├── package_augmentation │ ├── __init__.py │ └── poetry.py ├── package_identification │ ├── __init__.py │ └── poetry.py ├── __init__.py ├── config.py └── package.py ├── example ├── src │ └── my_package │ │ ├── resource │ │ └── my_package │ │ ├── my_package │ │ ├── __init__.py │ │ └── node.py │ │ ├── poetry.lock │ │ ├── package.xml │ │ └── pyproject.toml ├── README.md └── Dockerfile ├── setup.py ├── .gitignore ├── .github └── workflows │ └── release.yaml ├── setup.cfg ├── LICENSE └── README.md /colcon_poetry_ros/task/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /colcon_poetry_ros/dependencies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /colcon_poetry_ros/task/poetry/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/src/my_package/resource/my_package: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/src/my_package/my_package/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /colcon_poetry_ros/package_augmentation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /colcon_poetry_ros/package_identification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /colcon_poetry_ros/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.9.0" 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | 3 | *.pyc 4 | __pycache__/ 5 | *.egg-info/ 6 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | This is an example project that uses colcon-poetry-ros. We use Docker to 4 | illustrate setup, but the process is the same outside a container. 5 | 6 | ## Running 7 | 8 | Run the following commands _while in the project root_. 9 | 10 | ```bash 11 | docker build --tag colcon-poetry-ros-example --file example/Dockerfile . 12 | docker run -it colcon-poetry-ros-example 13 | ``` 14 | -------------------------------------------------------------------------------- /example/src/my_package/poetry.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. 2 | 3 | [[package]] 4 | name = "cowsay" 5 | version = "6.1" 6 | description = "The famous cowsay for GNU/Linux is now available for python" 7 | optional = false 8 | python-versions = ">=3.8" 9 | files = [ 10 | {file = "cowsay-6.1-py3-none-any.whl", hash = "sha256:274b1e6fc1b966d53976333eb90ac94cb07a450a700b455af9fbdf882244b30a"}, 11 | ] 12 | 13 | [metadata] 14 | lock-version = "2.0" 15 | python-versions = "^3.10" 16 | content-hash = "cfbcde678ffe8b6cf099e62eb4cee462f46dfe550f4af5748bef5d24d68405b8" 17 | -------------------------------------------------------------------------------- /example/src/my_package/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | my_package 5 | 1.0.0 6 | An example package that uses colcon-poetry-ros 7 | Urban Machine 8 | BSD-3-Clause 9 | 10 | rclpy 11 | std_msgs 12 | 13 | 14 | ament_python 15 | 16 | 17 | -------------------------------------------------------------------------------- /example/src/my_package/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "my_package" 3 | version = "1.0.0" 4 | description = "An example package that uses colcon-poetry-ros" 5 | authors = ["Urban Machine "] 6 | license = "BSD-3-Clause" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.10" 10 | cowsay = "^6.1" 11 | [tool.poetry.scripts] 12 | cowsayer = "my_package.node:main" 13 | 14 | [tool.colcon-poetry-ros.data-files] 15 | "share/ament_index/resource_index/packages" = ["resource/my_package"] 16 | "share/my_package" = ["package.xml"] 17 | 18 | [build-system] 19 | requires = ["poetry-core"] 20 | build-backend = "poetry.core.masonry.api" 21 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: 6 | - created 7 | 8 | jobs: 9 | pypi: 10 | runs-on: ubuntu-20.04 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Install Dependencies 14 | run: | 15 | sudo apt-get update 16 | sudo apt-get install python3-pip 17 | pip3 install build 18 | - name: Build 19 | run: python3 -m build --sdist --wheel --outdir dist/ 20 | shell: bash 21 | - name: Upload 22 | uses: pypa/gh-action-pypi-publish@master 23 | with: 24 | repository_url: https://upload.pypi.org/legacy/ 25 | password: ${{ secrets.PYPI_API_TOKEN }} 26 | -------------------------------------------------------------------------------- /example/src/my_package/my_package/node.py: -------------------------------------------------------------------------------- 1 | import cowsay 2 | import rclpy 3 | from rclpy.node import Node 4 | 5 | from std_msgs.msg import String 6 | 7 | 8 | class CowsayerNode(Node): 9 | """Sends cowsay text to a topic regularly""" 10 | 11 | def __init__(self): 12 | super().__init__("cowsayer") 13 | self.publisher_ = self.create_publisher(String, "topic", 10) 14 | self.timer = self.create_timer(1, self.timer_callback) 15 | 16 | def timer_callback(self): 17 | msg = String( 18 | data=cowsay.get_output_string("cow", "Hello world") 19 | ) 20 | self.publisher_.publish(msg) 21 | self.get_logger().info(f"Publishing: {msg.data}") 22 | 23 | 24 | def main(args=None): 25 | rclpy.init(args=args) 26 | 27 | cowsayer = CowsayerNode() 28 | 29 | rclpy.spin(cowsayer) 30 | 31 | cowsayer.destroy_node() 32 | rclpy.shutdown() 33 | 34 | 35 | if __name__ == '__main__': 36 | main() 37 | -------------------------------------------------------------------------------- /colcon_poetry_ros/task/poetry/test.py: -------------------------------------------------------------------------------- 1 | from colcon_core.logging import colcon_logger 2 | from colcon_core.plugin_system import satisfies_version 3 | from colcon_core.task import TaskExtensionPoint 4 | from colcon_core.task.python.test import PythonTestTask 5 | 6 | logger = colcon_logger.getChild(__name__) 7 | 8 | 9 | class PoetryTestTask(TaskExtensionPoint): 10 | """Adds support for testing via PyTest. Luckily we can reuse Colcon's built-in 11 | support for Python testing as long as we force it to always use PyTest. 12 | """ 13 | 14 | # Use a higher priority than PythonTestTask, since this one replaces that one 15 | PRIORITY = 250 16 | 17 | def __init__(self): 18 | super().__init__() 19 | satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') 20 | 21 | async def test(self): 22 | logger.info("Using the Poetry wrapper for PyTest support") 23 | 24 | # Force PythonTestTask to use PyTest, since the alternative is setup.py-based 25 | # testing 26 | # TODO: Is this true? It seems like the so-called "setup.py" test task just 27 | # invokes unittest, which should work fine. 28 | self.context.args.python_testing = "pytest" 29 | 30 | extension = PythonTestTask() 31 | extension.set_context(context=self.context) 32 | return await extension.test() 33 | -------------------------------------------------------------------------------- /example/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ROS_VERSION=humble 2 | FROM osrf/ros:${ROS_VERSION}-desktop 3 | 4 | ARG ROS_VERSION=humble 5 | ARG POETRY_VERSION=1.8.3 6 | ARG PIP_VERSION=24.3.1 7 | 8 | RUN apt-get update && apt-get install --yes python3-pip pipx python-is-python3 9 | RUN curl -fsSL https://install.python-poetry.org --output /tmp/install-poetry.py \ 10 | && POETRY_HOME=/usr/local python3 /tmp/install-poetry.py --version "${POETRY_VERSION}" 11 | RUN poetry self add poetry-plugin-bundle 12 | 13 | RUN pip3 install --upgrade pip==${PIP_VERSION} 14 | 15 | # Set up rosdep 16 | RUN rosdep update --rosdistro ${ROS_VERSION} 17 | 18 | # Install colcon-poetry-ros 19 | WORKDIR /colcon-poetry-ros 20 | COPY setup.cfg . 21 | COPY setup.py . 22 | COPY README.md . 23 | COPY colcon_poetry_ros colcon_poetry_ros 24 | RUN python3 -m pip install . 25 | 26 | WORKDIR /example 27 | 28 | # Install example rosdep dependencies 29 | COPY example/src/my_package/package.xml src/my_package/package.xml 30 | RUN rosdep install -i --from-path src --rosdistro "${ROS_VERSION}" -y 31 | 32 | # Install Poetry dependencies 33 | COPY example/src/my_package/pyproject.toml src/my_package/pyproject.toml 34 | COPY example/src/my_package/poetry.lock src/my_package/poetry.lock 35 | RUN python3 -m colcon_poetry_ros.dependencies.install --base-paths src 36 | 37 | COPY example/src src 38 | RUN colcon build 39 | 40 | CMD ["bash", "-c", "source install/setup.bash && ros2 run my_package cowsayer"] 41 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = colcon-poetry-ros 3 | version = attr: colcon_poetry_ros.__version__ 4 | url = https://github.com/UrbanMachine/colcon-poetry-ros 5 | author = Urban Machine 6 | author_email = info@urbanmachine.build 7 | maintainer = Urban Machine 8 | maintainer_email = info@urbanmachine.build 9 | classifiers = 10 | Development Status :: 3 - Alpha 11 | Environment :: Plugins 12 | Intended Audience :: Developers 13 | Programming Language :: Python 14 | Topic :: Software Development :: Build Tools 15 | License :: OSI Approved :: BSD License 16 | license = BSD-3-Clause 17 | license_files = LICENSE 18 | description = A Colcon extension providing support for Python projects that use Poetry 19 | long_description = file: README.md 20 | long_description_content_type = text/markdown 21 | keywords = colcon 22 | 23 | [options] 24 | packages = find: 25 | install_requires = 26 | colcon-core~=0.6 27 | toml~=0.10 28 | packaging 29 | setuptools 30 | zip_safe = true 31 | 32 | [options.entry_points] 33 | colcon_core.package_augmentation = 34 | poetry = colcon_poetry_ros.package_augmentation.poetry:PoetryPackageAugmentation 35 | colcon_core.package_identification = 36 | poetry = colcon_poetry_ros.package_identification.poetry:PoetryPackageIdentification 37 | colcon_core.task.build = 38 | poetry.python = colcon_poetry_ros.task.poetry.build:PoetryBuildTask 39 | colcon_core.task.test = 40 | poetry.python = colcon_poetry_ros.task.poetry.test:PoetryTestTask 41 | -------------------------------------------------------------------------------- /colcon_poetry_ros/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from distutils.util import strtobool 3 | from typing import TypeVar, Generic 4 | 5 | T = TypeVar('T') 6 | 7 | 8 | class _EnvironmentVariable(Generic[T]): 9 | """Manages configuration from environment variables, handling any parsing necessary 10 | to convert the variable from a string to the desired type. 11 | """ 12 | 13 | def __init__(self, name: str, default: T): 14 | """ 15 | :param name: The name of the environment variable 16 | :param default: The default value if the variable is not set 17 | """ 18 | self.name = name 19 | self.default = default 20 | 21 | def get(self) -> T: 22 | value = os.environ.get(self.name) 23 | if value is None: 24 | return self.default 25 | 26 | if isinstance(self.default, list): 27 | if value.strip() == "": 28 | return [] 29 | else: 30 | value_list = value.strip().split(",") 31 | return [v.strip() for v in value_list] 32 | elif isinstance(self.default, str): 33 | return value 34 | elif isinstance(self.default, bool): 35 | return strtobool(value) 36 | else: 37 | raise NotImplementedError( 38 | f"Unsupported environment variable type {type(self.default)}" 39 | ) 40 | 41 | 42 | run_depends_extras = _EnvironmentVariable("POETRY_RUN_DEPENDS_EXTRAS", []) 43 | test_depends_extras = _EnvironmentVariable("POETRY_TEST_DEPENDS_EXTRAS", ["test"]) 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2021, Urban Machine 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /colcon_poetry_ros/package_identification/poetry.py: -------------------------------------------------------------------------------- 1 | from colcon_core.package_descriptor import PackageDescriptor 2 | from colcon_core.package_identification import PackageIdentificationExtensionPoint, logger 3 | from colcon_core.plugin_system import satisfies_version 4 | 5 | from colcon_poetry_ros.package import PoetryPackage, NotAPoetryPackageError 6 | 7 | 8 | class PoetryPackageIdentification(PackageIdentificationExtensionPoint): 9 | """Identifies Python packages that use Poetry by referencing the pyproject.toml file 10 | """ 11 | 12 | # The priority needs to be higher than RosPackageIdentification and the built-in 13 | # Python identification. This identifier supersedes both. 14 | PRIORITY = 200 15 | 16 | def __init__(self): 17 | super().__init__() 18 | satisfies_version( 19 | PackageIdentificationExtensionPoint.EXTENSION_POINT_VERSION, 20 | "^1.0", 21 | ) 22 | 23 | def identify(self, desc: PackageDescriptor): 24 | if desc.type is not None and desc.type != "poetry.python": 25 | # Some other identifier claimed this package 26 | return 27 | 28 | try: 29 | project = PoetryPackage(desc.path, logger) 30 | except NotAPoetryPackageError: 31 | return 32 | 33 | if desc.name is not None and desc.name != project.name: 34 | raise RuntimeError( 35 | f"The {project.pyproject_file} file indicates that the package name is " 36 | f"'{project.name}', but the package name was already set as " 37 | f"'{desc.name}'" 38 | ) 39 | 40 | desc.type = "poetry.python" 41 | desc.name = project.name 42 | -------------------------------------------------------------------------------- /colcon_poetry_ros/package_augmentation/poetry.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import toml 4 | from colcon_core.package_augmentation import PackageAugmentationExtensionPoint 5 | from colcon_core.package_descriptor import PackageDescriptor 6 | from colcon_core.plugin_system import satisfies_version 7 | from colcon_core.package_augmentation.python import \ 8 | create_dependency_descriptor, logger 9 | 10 | from colcon_poetry_ros import config 11 | from colcon_poetry_ros.package_identification.poetry import PoetryPackage 12 | 13 | 14 | class PoetryPackageAugmentation(PackageAugmentationExtensionPoint): 15 | """Augment Python packages that use Poetry by referencing the pyproject.toml file""" 16 | 17 | _TOOL_SECTION = "tool" 18 | _COLCON_POETRY_ROS_SECTION = "colcon-poetry-ros" 19 | _DEPENDENCIES_SECTION = "dependencies" 20 | _DEPEND_LIST = "depend" 21 | _BUILD_DEPEND_LIST = "build_depend" 22 | _EXEC_DEPEND_LIST = "exec_depend" 23 | _TEST_DEPEND_LIST = "test_depend" 24 | _PACKAGE_BUILD_CATEGORY = "build" 25 | _PACKAGE_EXEC_CATEGORY = "run" 26 | _PACKAGE_TEST_CATEGORY = "test" 27 | 28 | def __init__(self): 29 | super().__init__() 30 | satisfies_version( 31 | PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, 32 | "^1.0", 33 | ) 34 | 35 | def augment_package( 36 | self, desc: PackageDescriptor, *, additional_argument_names=None 37 | ): 38 | if desc.type != "poetry.python": 39 | # Some other identifier claimed this package 40 | return 41 | 42 | project = PoetryPackage(desc.path, logger) 43 | project.check_lock_file_exists() 44 | 45 | if not shutil.which("poetry"): 46 | raise RuntimeError( 47 | "Could not find the poetry command. Is Poetry installed?" 48 | ) 49 | 50 | pyproject_toml = desc.path / "pyproject.toml" 51 | pyproject = toml.loads(pyproject_toml.read_text()) 52 | 53 | if not(self._TOOL_SECTION in pyproject and 54 | self._COLCON_POETRY_ROS_SECTION in pyproject[self._TOOL_SECTION] and 55 | self._DEPENDENCIES_SECTION in pyproject[self._TOOL_SECTION][self._COLCON_POETRY_ROS_SECTION]): 56 | return 57 | colcon_deps = pyproject[self._TOOL_SECTION][self._COLCON_POETRY_ROS_SECTION][self._DEPENDENCIES_SECTION] 58 | # Parses dependencies to other colcon packages indicated in the pyproject.toml file. 59 | if self._BUILD_DEPEND_LIST in colcon_deps: 60 | build_depend = set(colcon_deps[self._BUILD_DEPEND_LIST]) 61 | else: 62 | build_depend = set() 63 | 64 | if self._EXEC_DEPEND_LIST in colcon_deps: 65 | exec_depend = set(colcon_deps[self._EXEC_DEPEND_LIST]) 66 | else: 67 | exec_depend = set() 68 | 69 | if self._TEST_DEPEND_LIST in colcon_deps: 70 | test_depend = set(colcon_deps[self._TEST_DEPEND_LIST]) 71 | else: 72 | test_depend = set() 73 | 74 | # Depend add the deps to the build and exec depends 75 | if self._DEPEND_LIST in colcon_deps: 76 | depends = colcon_deps[self._DEPEND_LIST] 77 | build_depend.update(depends) 78 | exec_depend.update(depends) 79 | 80 | desc.dependencies[self._PACKAGE_BUILD_CATEGORY] = set( 81 | create_dependency_descriptor(dep) for dep in build_depend 82 | ) 83 | desc.dependencies[self._PACKAGE_EXEC_CATEGORY] = set( 84 | create_dependency_descriptor(dep) for dep in exec_depend 85 | ) 86 | desc.dependencies[self._PACKAGE_TEST_CATEGORY] = set( 87 | create_dependency_descriptor(dep) for dep in test_depend 88 | ) 89 | -------------------------------------------------------------------------------- /colcon_poetry_ros/dependencies/install.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | import sys 4 | from typing import List 5 | import subprocess 6 | import logging 7 | 8 | from colcon_poetry_ros.package_identification.poetry import ( 9 | PoetryPackage, 10 | NotAPoetryPackageError, 11 | ) 12 | 13 | 14 | def main(): 15 | args = _parse_args() 16 | logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) 17 | 18 | for project in _discover_packages(args.base_paths): 19 | logging.info(f"Installing dependencies for {project.path.name}...") 20 | _install_dependencies( 21 | project, args.install_base, args.merge_install 22 | ) 23 | 24 | logging.info("\nDependencies installed!") 25 | 26 | 27 | def _discover_packages(base_paths: List[Path]) -> List[PoetryPackage]: 28 | projects: List[PoetryPackage] = [] 29 | 30 | potential_packages = [] 31 | for path in base_paths: 32 | potential_packages += list(path.glob("*")) 33 | 34 | for path in potential_packages: 35 | if path.is_dir(): 36 | try: 37 | project = PoetryPackage(path) 38 | except NotAPoetryPackageError: 39 | continue 40 | else: 41 | projects.append(project) 42 | 43 | if len(projects) == 0: 44 | base_paths_str = ", ".join([str(p) for p in base_paths]) 45 | logging.error( 46 | f"No packages were found in the following paths: {base_paths_str}" 47 | ) 48 | sys.exit(1) 49 | 50 | return projects 51 | 52 | 53 | def _install_dependencies( 54 | project: PoetryPackage, install_base: Path, merge_install: bool 55 | ) -> None: 56 | """Uses poetry-bundle-plugin to create a virtual environment with all the project's 57 | dependencies in Colcon's install directory. We need to use the bundle plugin because 58 | Poetry does not natively let us install projects to a custom location. 59 | """ 60 | try: 61 | subprocess.run( 62 | ["poetry", "bundle", "venv", "--help"], 63 | check=True, 64 | stdout=subprocess.DEVNULL, 65 | stderr=subprocess.DEVNULL, 66 | ) 67 | except subprocess.CalledProcessError: 68 | logging.error( 69 | "The Poetry bundle plugin does not appear to be installed! See the " 70 | "project page for installation instructions: " 71 | "https://github.com/python-poetry/poetry-plugin-bundle" 72 | ) 73 | sys.exit(1) 74 | 75 | if not merge_install: 76 | install_base /= project.name 77 | 78 | subprocess.run( 79 | [ 80 | "poetry", 81 | "bundle", 82 | "venv", 83 | "--no-interaction", 84 | str(install_base.absolute()), 85 | ], 86 | check=True, 87 | cwd=project.path 88 | ) 89 | 90 | _enable_system_site_packages(install_base) 91 | 92 | 93 | def _enable_system_site_packages(install_base: Path) -> None: 94 | """Enables system site packages for a virtual environment. This allows system 95 | packages to be imported from within the virtual environment, which is necessary 96 | since ROS depends on some Python packages supplied by APT. 97 | 98 | Based on this: https://stackoverflow.com/a/40972692/2159348 99 | 100 | :param install_base: The location of the virtual environment 101 | """ 102 | pyvenv_config_file = install_base / "pyvenv.cfg" 103 | 104 | # Even though the pyvenv.cfg file looks like a ConfigParser-style ini file, it's not 105 | # because it's missing sections, so we're stuck doing string manipulation instead 106 | old_config = pyvenv_config_file.read_text() 107 | new_config = "" 108 | for line in old_config.splitlines(): 109 | if line.startswith("include-system-site-packages"): 110 | new_config += "include-system-site-packages = true\n" 111 | else: 112 | new_config += f"{line}\n" 113 | 114 | pyvenv_config_file.write_text(new_config) 115 | 116 | 117 | def _parse_args() -> argparse.Namespace: 118 | parser = argparse.ArgumentParser( 119 | description="Searches for Poetry packages and installs their dependencies " 120 | "to a configurable install base" 121 | ) 122 | 123 | parser.add_argument( 124 | "--base-paths", 125 | nargs="+", 126 | type=Path, 127 | default=[Path.cwd()], 128 | help="The paths to start looking for Poetry projects in. Defaults to the " 129 | "current directory." 130 | ) 131 | 132 | parser.add_argument( 133 | "--install-base", 134 | type=Path, 135 | default=Path("install"), 136 | help="The base path for all install prefixes (default: install)", 137 | ) 138 | 139 | parser.add_argument( 140 | "--merge-install", 141 | action="store_true", 142 | help="Merge all install prefixes into a single location", 143 | ) 144 | 145 | parser.add_argument( 146 | "-v", 147 | "--verbose", 148 | action="store_true", 149 | help="If provided, debug logs will be printed", 150 | ) 151 | 152 | return parser.parse_args() 153 | 154 | 155 | if __name__ == "__main__": 156 | main() 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # colcon-poetry-ros 2 | 3 | An extension for [colcon-core][colcon-core] that adds support for Python 4 | packages that use [Poetry][poetry] within ROS. This extension is a replacement 5 | for Colcon's built-in `setup.cfg` based Python support and the Python-related 6 | bits in [colcon-ros][colcon-ros]. 7 | 8 | We use this extension with Humble, but other versions should work as well. 9 | Please create an issue if you see problems! 10 | 11 | [colcon-core]: https://github.com/colcon/colcon-core 12 | [poetry]: https://python-poetry.org/ 13 | [colcon-ros]: https://github.com/colcon/colcon-ros 14 | 15 | ## Getting Started 16 | 17 | 1. [Install Poetry][installing poetry] and the 18 | [Poetry Bundle plugin][installing poetry bundle], if you haven't already. 19 | 20 | 2. Install this extension with Pip: 21 | 22 | ```bash 23 | pip3 install colcon-poetry-ros 24 | ``` 25 | 26 | 3. Add a `pyproject.toml` in the root of your package's directory. Each 27 | package should have its own `pyproject.toml` file. It should look something 28 | like this: 29 | 30 | ```toml 31 | [tool.poetry] 32 | name = "my_package" 33 | version = "0.1.0" 34 | description = "Does something cool" 35 | authors = ["John Smith "] 36 | license = "BSD-3-Clause" 37 | 38 | [tool.poetry.dependencies] 39 | python = "^3.8" 40 | 41 | [tool.poetry.scripts] 42 | node_a = "my_package.node_a:main" 43 | node_b = "my_package.node_b:main" 44 | 45 | [tool.colcon-poetry-ros.data-files] 46 | "share/ament_index/resource_index/packages" = ["resource/my_package"] 47 | "share/my_package" = ["package.xml"] 48 | 49 | [build-system] 50 | requires = ["poetry-core>=1.0.0"] 51 | build-backend = "poetry.core.masonry.api" 52 | ``` 53 | 54 | 4. Install your packages' Python dependencies using a script included with 55 | this plugin: 56 | 57 | ```bash 58 | python3 -m colcon_poetry_ros.dependencies.install --base-paths 59 | ``` 60 | 61 | 5. Finally, run your build like normal: 62 | 63 | ```bash 64 | colcon build 65 | ``` 66 | 67 | ## Testing 68 | 69 | This extension currently supports projects based on PyTest. Run the following 70 | command to start tests: 71 | 72 | ```bash 73 | colcon test 74 | ``` 75 | 76 | ## Node Entrypoints 77 | 78 | If you want to be able to run your nodes using `ros2 run`, add your node's 79 | entrypoint to the `tool.poetry.scripts` table. See 80 | [Poetry's documentation][poetry-scripts] for details. 81 | 82 | ```toml 83 | [tool.poetry.scripts] 84 | node_a = "my_package.node_a:main" 85 | node_b = "my_package.node_b:main" 86 | ``` 87 | 88 | [poetry-scripts]: https://python-poetry.org/docs/pyproject/#scripts 89 | 90 | ## Data Files 91 | 92 | Poetry has only limited support for including data files in an installation, 93 | and the current implementation is not flexible enough to be used with ROS. 94 | Instead, this extension consults a custom section in your `pyproject.toml`, 95 | called `tool.colcon-poetry-ros.data-files`. 96 | 97 | The format is intended to be mostly identical to the `data_files` field used 98 | by [setuptools][setuptools-data-files]. The main differences are that copying 99 | entire directories is supported, and globbing is not yet implemented. 100 | 101 | All ROS packages must have, at minimum, these entries in the 102 | `tool.colcon-poetry-ros.data-files` section (with `{package_name}` replaced 103 | with the name of your package): 104 | 105 | ```toml 106 | [tool.colcon-poetry-ros.data-files] 107 | "share/ament_index/resource_index/packages" = ["resource/{package_name}"] 108 | "share/{package_name}" = ["package.xml"] 109 | ``` 110 | 111 | These entries take care of adding the package index marker and `package.xml` 112 | file to the installation. 113 | 114 | [setuptools-data-files]: https://setuptools.pypa.io/en/latest/userguide/datafiles.html 115 | 116 | ## Python Dependency Details 117 | 118 | Poetry dependencies are not installed as part of the build process, so they 119 | must be installed using a separate tool that's included in this package. 120 | 121 | ```bash 122 | python3 -m colcon_poetry_ros.dependencies.install --base-paths 123 | ``` 124 | 125 | This command creates a virtual environment within Colcon's base install 126 | directory, then installs each package's dependencies in that virtual 127 | environment. 128 | 129 | If you customize `colcon build` with the `--install-base` or `--merge-install` 130 | flags, make sure to provide those to this tool as well. 131 | 132 | We split dependency installation out of `colcon build` to make development 133 | iterations faster. Third-party dependencies change less frequency than first 134 | party code, so it's often a waste of time to resolve dependencies on every 135 | iteration. This is especially elegant in container-based workflows, an example 136 | of which can be found in the `examples/` directory. 137 | 138 | ## Communicating Package Dependencies to Colcon 139 | 140 | Colcon can be given information on dependencies between packages, which 141 | affects build order and can be displayed in tools like `colcon graph`. These 142 | dependencies can be explicitly defined in the `pyproject.toml` under a custom 143 | section called `tool.colcon-poetry-ros.dependencies`. 144 | 145 | ```toml 146 | [tool.colcon-poetry-ros.dependencies] 147 | depend = ["foo_package"] # This will add to both `build_depend` and `exec_depend` following `package.xml` standards 148 | build_depend = ["bar_package"] 149 | exec_depend = ["baz_package"] 150 | test_depend = ["qux_package"] 151 | ``` 152 | -------------------------------------------------------------------------------- /colcon_poetry_ros/package.py: -------------------------------------------------------------------------------- 1 | import re 2 | import subprocess 3 | from pathlib import Path 4 | import logging 5 | from tempfile import NamedTemporaryFile 6 | from typing import List, Set 7 | 8 | import toml 9 | from packaging.version import VERSION_PATTERN 10 | 11 | PACKAGE_NAME_PATTERN = r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$" 12 | """Matches on valid package names when run with re.IGNORECASE. 13 | Pulled from: https://peps.python.org/pep-0508/#names 14 | """ 15 | 16 | 17 | class NotAPoetryPackageError(Exception): 18 | """The given directory does not point to a Poetry project""" 19 | 20 | 21 | class PoetryPackage: 22 | """Contains information on a package defined with Poetry""" 23 | 24 | def __init__(self, path: Path, logger: logging.Logger = logging): 25 | """ 26 | :param path: The root path of the Poetry project 27 | :param logger: A logger to log with! 28 | """ 29 | self.path = path 30 | self.logger = logger 31 | 32 | self.pyproject_file = path / "pyproject.toml" 33 | if not self.pyproject_file.is_file(): 34 | # Poetry requires a pyproject.toml to function 35 | raise NotAPoetryPackageError() 36 | 37 | try: 38 | self.pyproject = toml.loads(self.pyproject_file.read_text()) 39 | except toml.TomlDecodeError as ex: 40 | raise RuntimeError( 41 | f"Failed to parse {self.pyproject_file} as a TOML file: {ex}" 42 | ) 43 | 44 | if "tool" not in self.pyproject or "poetry" not in self.pyproject["tool"]: 45 | logger.debug( 46 | f"The {self.pyproject_file} file does not have a [tool.poetry] " 47 | f"section. The file is likely there to instruct a tool other than " 48 | f"Poetry." 49 | ) 50 | raise NotAPoetryPackageError() 51 | 52 | logger.info(f"Project {path} appears to be a Poetry ROS project") 53 | 54 | poetry_config = self.pyproject["tool"]["poetry"] 55 | 56 | if "name" not in poetry_config: 57 | raise RuntimeError( 58 | f"Failed to determine Python package name in {self.path}: The " 59 | f"[tool.poetry] section must have a 'name' field" 60 | ) 61 | 62 | self.name = poetry_config["name"] 63 | 64 | def check_lock_file_exists(self) -> Path: 65 | """Raises an exception if the lock file is not available. 66 | 67 | :return: The lock file location 68 | """ 69 | lock_file = self.path / "poetry.lock" 70 | if not lock_file.is_file(): 71 | raise RuntimeError( 72 | f"The lock file is missing, expected at '{lock_file}'. Have you run " 73 | f"'poetry lock'?" 74 | ) 75 | return lock_file 76 | 77 | def get_requirements_txt(self, extras: List[str]) -> str: 78 | """Generates a list of the project's dependencies in requirements.txt format. 79 | 80 | :param extras: Names of extras whose dependencies should be included 81 | :return: The requirements.txt text 82 | """ 83 | command = [ 84 | "poetry", 85 | "export", 86 | "--format", 87 | "requirements.txt", 88 | ] 89 | 90 | for extra in extras: 91 | command += ["--extras", extra] 92 | 93 | # Create a temporary file for `poetry export` to write its output to. We can't 94 | # just capture stdout because Poetry 1.2 uses stdout for logging, too. 95 | with NamedTemporaryFile("r") as requirements_file: 96 | command += ["--output", requirements_file.name] 97 | 98 | try: 99 | subprocess.run( 100 | command, 101 | cwd=self.path, 102 | check=True, 103 | encoding="utf-8", 104 | ) 105 | except subprocess.CalledProcessError as ex: 106 | raise RuntimeError( 107 | f"Failed to export Poetry dependencies in the requirements.txt " 108 | f"format: {ex}" 109 | ) 110 | 111 | return requirements_file.read() 112 | 113 | def get_dependencies(self, extras: List[str]) -> Set[str]: 114 | """Gets dependencies for a Poetry project. 115 | 116 | :param extras: Names of extras whose dependencies should be included 117 | :return: A list of dependencies in PEP440 format 118 | """ 119 | try: 120 | result = subprocess.run( 121 | ["poetry", "show", "--no-interaction"], 122 | cwd=self.path, 123 | check=True, 124 | stdout=subprocess.PIPE, 125 | encoding="utf-8", 126 | ) 127 | except subprocess.CalledProcessError as ex: 128 | raise RuntimeError(f"Failed to read package dependencies: {ex}") 129 | 130 | dependencies = set() 131 | 132 | for line in result.stdout.splitlines(): 133 | try: 134 | dependency = self._parse_dependency_line(line) 135 | except ValueError as ex: 136 | self.logger.warning(str(ex)) 137 | else: 138 | dependencies.add(dependency) 139 | 140 | return dependencies 141 | 142 | def _parse_dependency_line(self, line: str) -> str: 143 | """Makes a best-effort attempt to parse lines from ``poetry show`` as 144 | dependencies. Poetry does not have a stable CLI interface, so this logic may 145 | not be sufficient now or in the future. A smarter approach is needed. 146 | 147 | :param line: A raw line from ``poetry show`` 148 | :return: A dependency string in PEP440 format 149 | """ 150 | 151 | components = line.split() 152 | if len(components) < 2: 153 | raise ValueError(f"Could not parse line '{line}' as a dependency") 154 | 155 | name = components[0] 156 | if re.match(PACKAGE_NAME_PATTERN, name, re.IGNORECASE) is None: 157 | raise ValueError(f"Invalid dependency name '{name}'") 158 | 159 | version = None 160 | 161 | # Search for an item that looks like a version. Poetry adds other data in front 162 | # of the version number under certain circumstances. 163 | for item in components[1:]: 164 | if re.match(VERSION_PATTERN, item, re.VERBOSE | re.IGNORECASE) is not None: 165 | version = item 166 | 167 | if version is None: 168 | raise ValueError( 169 | f"For dependency '{name}': Could not find version specification " 170 | f"in '{line}'" 171 | ) 172 | 173 | return f"{name}=={version}" 174 | -------------------------------------------------------------------------------- /colcon_poetry_ros/task/poetry/build.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | import toml 5 | from colcon_core.environment import create_environment_hooks, \ 6 | create_environment_scripts 7 | from colcon_core.logging import colcon_logger 8 | from colcon_core.plugin_system import satisfies_version 9 | from colcon_core.shell import get_command_environment, create_environment_hook 10 | from colcon_core.task import TaskExtensionPoint 11 | from colcon_core.task import run 12 | 13 | from colcon_poetry_ros import config 14 | 15 | 16 | logger = colcon_logger.getChild(__name__) 17 | 18 | 19 | class PoetryBuildTask(TaskExtensionPoint): 20 | """Builds Python packages using Poetry""" 21 | 22 | def __init__(self): 23 | super().__init__() 24 | satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') 25 | 26 | async def build(self, *, additional_hooks=None): 27 | pkg = self.context.pkg 28 | args = self.context.args 29 | 30 | if pkg.type != "poetry.python": 31 | logger.error( 32 | f"The Poetry build was invoked on the wrong package type! Expected " 33 | f"'poetry.python' but got '{pkg.type}'." 34 | ) 35 | 36 | logger.info(f"Building Poetry Python package in '{args.path}'") 37 | 38 | try: 39 | env = await get_command_environment( 40 | "build", args.build_base, self.context.dependencies 41 | ) 42 | except RuntimeError as e: 43 | logger.error(str(e)) 44 | return 1 45 | 46 | # In this step, we want to install the ROS package's source without touching 47 | # its dependencies, because those are managed by 48 | # colcon_poetry_ros.dependencies.install. Poetry (even with bundle) doesn't have 49 | # a way to re-install only the source code to a target directory, so instead we 50 | # export the project as a wheel and use Pip to install the source instead. 51 | # Related: https://github.com/python-poetry/poetry-plugin-bundle/issues/87 52 | completed = await run( 53 | self.context, 54 | ["poetry", "build", "--format", "wheel"], 55 | cwd=args.path, 56 | env=env, 57 | ) 58 | if completed.returncode: 59 | logger.error(f"Poetry failed to build the package for '{args.path}'") 60 | return completed.returncode 61 | 62 | poetry_dist = Path(args.build_base) / "poetry_dist" 63 | if poetry_dist.exists(): 64 | shutil.rmtree(str(poetry_dist)) 65 | 66 | shutil.move( 67 | str(Path(args.path) / "dist"), 68 | str(poetry_dist) 69 | ) 70 | 71 | # Find the wheel file that Poetry generated 72 | wheels = list(poetry_dist.glob("*.whl")) 73 | if len(wheels) == 0: 74 | logger.error(f"Poetry failed to produce a wheel file in '{poetry_dist}'") 75 | wheel_name = str(wheels[0]) 76 | 77 | # Include any extras that are needed at runtime. Extras are included by adding 78 | # a bracket-surrounded comma-separated list to the end of the package name, like 79 | # "colcon-poetry-ros[cool_stuff,other_stuff]" 80 | extras = config.run_depends_extras.get() 81 | if len(extras) > 0: 82 | extras_str = ",".join(extras) 83 | extras_str = f"[{extras_str}]" 84 | wheel_name += extras_str 85 | 86 | venv_python_executable = Path(args.install_base) / "bin/python" 87 | if not venv_python_executable.exists(): 88 | logger.error( 89 | f"No virtual environment exists in {args.install_base}. Have " 90 | f"dependencies not been installed yet?" 91 | ) 92 | return 1 93 | 94 | # Install Poetry's generated wheel 95 | completed = await run( 96 | self.context, 97 | [ 98 | str(venv_python_executable), 99 | "-m", 100 | "pip", 101 | "install", 102 | wheel_name, 103 | # pip will skip installation if the package version is the same 104 | # but we want the installed version to always reflect the source 105 | # regardless of the package version 106 | "--force-reinstall", 107 | # Turns off Pip's check to ensure installed binaries are in the 108 | # PATH. ROS workspaces take care of setting the PATH, but Pip 109 | # doesn't know that. 110 | "--no-warn-script-location", 111 | "--no-deps", 112 | ], 113 | cwd=args.path, 114 | env=env, 115 | ) 116 | if completed.returncode: 117 | logger.error(f"Failed to install Poetry's wheel for '{args.path}'") 118 | return completed.returncode 119 | 120 | # Poetry installs scripts to {prefix}/bin, but ROS wants them at 121 | # {prefix}/lib/{package_name} 122 | poetry_script_dir = Path(args.install_base) / "bin" 123 | if not poetry_script_dir.exists(): 124 | logger.info(f"Attempting to use .../local/bin instead for poetry_script_dir") 125 | poetry_script_dir = Path(args.install_base) / "local" / "bin" 126 | ros_script_dir = Path(args.install_base) / "lib" / pkg.name 127 | if poetry_script_dir.is_dir(): 128 | ros_script_dir.mkdir(parents=True, exist_ok=True) 129 | 130 | script_files = poetry_script_dir.glob("*") 131 | script_files = filter(Path.is_file, script_files) 132 | 133 | for script in script_files: 134 | shutil.copy2(str(script), str(ros_script_dir)) 135 | else: 136 | logger.warning( 137 | "Poetry did not install any scripts. Are you missing a " 138 | "[tool.poetry.scripts] section?" 139 | ) 140 | 141 | return_code = await self._add_data_files() 142 | if return_code != 0: 143 | return return_code 144 | 145 | # This hook is normally defined by AmentPythonBuildTask, but since this class 146 | # replaces that, we have to define it ourselves 147 | additional_hooks = create_environment_hook( 148 | "ament_prefix_path", 149 | Path(args.install_base), 150 | self.context.pkg.name, 151 | "AMENT_PREFIX_PATH", 152 | "", 153 | mode="prepend", 154 | ) 155 | 156 | hooks = create_environment_hooks(args.install_base, pkg.name) 157 | create_environment_scripts( 158 | pkg, args, default_hooks=list(hooks), additional_hooks=additional_hooks 159 | ) 160 | 161 | async def _add_data_files(self) -> int: 162 | """Installs data files based on the [tool.colcon-poetry-ros.data-files] table. 163 | Poetry's support for data files is fairly incomplete at the time of writing, so 164 | we need to do this ourselves. 165 | 166 | See: https://github.com/python-poetry/poetry/issues/2015 167 | """ 168 | pkg = self.context.pkg 169 | args = self.context.args 170 | 171 | pyproject_toml = pkg.path / "pyproject.toml" 172 | pyproject = toml.loads(pyproject_toml.read_text()) 173 | 174 | try: 175 | data_files = pyproject["tool"]["colcon-poetry-ros"]["data-files"] 176 | except KeyError: 177 | logger.warning( 178 | f"File {pyproject_toml} does not define any data files, so none will " 179 | f"be included in the installation" 180 | ) 181 | return 0 182 | 183 | if not isinstance(data_files, dict): 184 | logger.error(f"{_DATA_FILES_TABLE} must be a table") 185 | return 1 186 | 187 | for destination, sources in data_files.items(): 188 | if not isinstance(sources, list): 189 | logger.error( 190 | f"Field '{destination}' in {_DATA_FILES_TABLE} must be an array" 191 | ) 192 | 193 | dest_path = Path(args.install_base) / destination 194 | dest_path.mkdir(parents=True, exist_ok=True) 195 | 196 | for source in sources: 197 | source_path = pkg.path / Path(source) 198 | _copy_path(source_path, dest_path) 199 | 200 | return 0 201 | 202 | 203 | def _copy_path(src: Path, dest: Path) -> None: 204 | """Copies a file or directory to a destination""" 205 | if src.is_dir(): 206 | # The provided destination is interpreted as a parent directory when copying 207 | # directories 208 | dest_dir = dest / src.name 209 | # shutil.copytree can not completely overwrite a directory, so we need to delete 210 | # the existing one in advance 211 | if dest_dir.exists(): 212 | _delete_path(dest_dir) 213 | 214 | shutil.copytree(str(src), str(dest_dir)) 215 | else: 216 | shutil.copy2(str(src), str(dest)) 217 | 218 | 219 | def _delete_path(path: Path) -> None: 220 | """Deletes a file or directory""" 221 | if path.is_file(): 222 | path.unlink() 223 | elif path.is_dir(): 224 | shutil.rmtree(str(path)) 225 | 226 | 227 | _DATA_FILES_TABLE = "[tool.colcon-poetry-ros.data-files]" 228 | --------------------------------------------------------------------------------