├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── circuitpython_build_tools ├── build.py ├── scripts │ ├── build_bundles.py │ ├── build_mpy_cross.py │ └── circuitpython_mpy_cross.py └── target_versions.py ├── requirements.txt └── setup.py /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries 2 | # SPDX-FileCopyrightText: 2021 James Carr 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | name: Build CI 7 | 8 | on: [pull_request, push] 9 | 10 | jobs: 11 | build-and-test: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Dump GitHub context 15 | env: 16 | GITHUB_CONTEXT: ${{ toJson(github) }} 17 | run: echo "$GITHUB_CONTEXT" 18 | - name: Set up Python 3.12 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: "3.12" 22 | - name: Versions 23 | run: | 24 | python3 --version 25 | - name: Checkout Current Repo 26 | uses: actions/checkout@v4 27 | with: 28 | filter: 'blob:none' 29 | depth: 0 30 | - name: Install requirements 31 | run: | 32 | sudo apt-get update 33 | sudo apt-get install libudev-dev libusb-1.0 34 | sudo apt-get install -y gettext 35 | pip install -r requirements.txt 36 | - name: Library version 37 | run: git describe --dirty --always --tags 38 | - name: Install package locally 39 | run: pip install -e . 40 | - name: Test building single package 41 | run: | 42 | git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git 43 | cd Adafruit_CircuitPython_FeatherWing 44 | circuitpython-build-bundles --filename_prefix test-single --library_location . 45 | - name: Test building bundle 46 | run: | 47 | # Use the community bundle because it's smaller and faster 48 | git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git 49 | cd CircuitPython_Community_Bundle 50 | circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2 51 | - name: Build Python package 52 | run: | 53 | pip install --upgrade setuptools wheel twine readme_renderer testresources 54 | python setup.py sdist 55 | twine check dist/* 56 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries 2 | # SPDX-FileCopyrightText: 2021 James Carr 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | name: Release Actions 7 | 8 | on: 9 | release: 10 | types: [published] 11 | 12 | jobs: 13 | upload-pypi: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | with: 18 | filter: 'blob:none' 19 | depth: 0 20 | - name: Set up Python 21 | uses: actions/setup-python@v5 22 | with: 23 | python-version: '3.12' 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install setuptools wheel twine 28 | - name: Build and publish 29 | env: 30 | TWINE_USERNAME: ${{ secrets.pypi_username }} 31 | TWINE_PASSWORD: ${{ secrets.pypi_password }} 32 | run: | 33 | python setup.py sdist 34 | twine upload dist/* 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.zip 3 | MANIFEST 4 | build_deps 5 | dist 6 | *.egg-info 7 | circuitpython_build_tools/data/ 8 | .eggs 9 | version.py 10 | .env/* 11 | .DS_Store 12 | .idea/* 13 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at support@adafruit.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Scott Shawcroft for Adafruit Industries 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Adafruit CircuitPython Build Tools 2 | 3 | [![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://adafru.it/discord) 4 | 5 | This repo contains build scripts used to build the 6 | [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle) 7 | and individual library release zips. Its focused on Github Actions support but will also work locally 8 | when a gcc compiler is present. 9 | 10 | The scripts will either fetch a pre-built mpy-cross from s3 or 11 | automatically clone the [CircuitPython repo](https://github.com/adafruit/circuitpython) and attempt 12 | to build mpy-cross. You'll need some version of gcc for this to work. 13 | 14 | ## Setting up libraries 15 | 16 | These build tools automatically build .mpy files and zip them up for 17 | CircuitPython when a new tagged release is created. To add support to a repo 18 | you need to use the [CircuitPython 19 | cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) 20 | to generate `.github/workflows/*.yml`. 21 | 22 | The bundle build will produce one zip file for every major CircuitPython 23 | release supported containing compatible mpy files and a zip with human readable py files. 24 | It'll also "release" a `z-build_tools_version-x.x.x.ignore` file that will be 25 | used to determine when a library needs new release files because the build tools 26 | themselves changed, such as when a new major CircuitPython release happens. 27 | 28 | ## Building libraries locally 29 | 30 | To build libraries built with the build tools you'll need to install the 31 | circuitpython-build-tools package. 32 | 33 | ```shell 34 | python3 -m venv .env 35 | source .env/bin/activate 36 | pip install circuitpython-build-tools 37 | circuitpython-build-bundles --filename_prefix --library_location . 38 | ``` 39 | 40 | When making changes to `circuitpython-build-tools` itself, you can test your changes 41 | locally like so: 42 | 43 | ```shell 44 | cd circuitpython-build-tools # this will be specific to your storage location 45 | python3 -m venv .env 46 | source .env/bin/activate 47 | pip install -e . # '-e' is pip's "development" install feature 48 | circuitpython-build-bundles --filename_prefix --library_location 49 | ``` 50 | 51 | ## Contributing 52 | 53 | Contributions are welcome! Please read our [Code of Conduct] 54 | (https://github.com/adafruit/Adafruit\_CircuitPython\_adabot/blob/master/CODE\_OF\_CONDUCT.md) 55 | before contributing to help this project stay welcoming. 56 | -------------------------------------------------------------------------------- /circuitpython_build_tools/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries 6 | # 2018, 2019 Michael Schroeder 7 | # 2021 James Carr 8 | # 9 | # Permission is hereby granted, free of charge, to any person obtaining a copy 10 | # of this software and associated documentation files (the "Software"), to deal 11 | # in the Software without restriction, including without limitation the rights 12 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | # copies of the Software, and to permit persons to whom the Software is 14 | # furnished to do so, subject to the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be included in 17 | # all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 25 | # THE SOFTWARE. 26 | 27 | import functools 28 | import multiprocessing 29 | import os 30 | import os.path 31 | import platform 32 | import pathlib 33 | import re 34 | import requests 35 | import semver 36 | import shutil 37 | import stat 38 | import sys 39 | import subprocess 40 | import tempfile 41 | import platformdirs 42 | 43 | @functools.cache 44 | def _git_version(): 45 | version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace") 46 | version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0) 47 | return tuple(int(part) for part in version_str.split(".")) 48 | 49 | def git_filter_arg(): 50 | clone_supports_filter = ( 51 | False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0) 52 | ) 53 | 54 | if clone_supports_filter: 55 | return ["--filter=blob:none"] 56 | else: 57 | return [] 58 | 59 | # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! 60 | # and should be removed when the fixed version is incorporated in its respective bundle. 61 | 62 | pyproject_py_modules_blocklist = set(( 63 | # community bundle 64 | "at24mac_eeprom", 65 | "p1am_200_helpers", 66 | )) 67 | 68 | if sys.version_info >= (3, 11): 69 | from tomllib import loads as load_toml 70 | else: 71 | from tomli import loads as load_toml 72 | 73 | mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) 74 | 75 | def load_pyproject_toml(lib_path: pathlib.Path): 76 | try: 77 | return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) 78 | except FileNotFoundError: 79 | print(f"No pyproject.toml in {lib_path}") 80 | return {} 81 | 82 | def get_nested(doc, *args, default=None): 83 | for a in args: 84 | if doc is None: return default 85 | try: 86 | doc = doc[a] 87 | except (KeyError, IndexError) as e: 88 | return default 89 | return doc 90 | 91 | IGNORE_PY = ["setup.py", "conf.py", "__init__.py"] 92 | GLOB_PATTERNS = ["*.py", "*.bin"] 93 | S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross" 94 | 95 | def version_string(path=None, *, valid_semver=False): 96 | version = None 97 | tag = subprocess.run('git describe --tags --exact-match', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path) 98 | if tag.returncode == 0: 99 | version = tag.stdout.strip().decode("utf-8", "strict") 100 | else: 101 | describe = subprocess.run("git describe --tags --always", shell=True, stdout=subprocess.PIPE, cwd=path) 102 | describe = describe.stdout.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) 103 | if len(describe) == 3: 104 | tag, additional_commits, commitish = describe 105 | commitish = commitish[1:] 106 | else: 107 | tag = "0.0.0" 108 | commit_count = subprocess.run("git rev-list --count HEAD", shell=True, stdout=subprocess.PIPE, cwd=path) 109 | additional_commits = commit_count.stdout.strip().decode("utf-8", "strict") 110 | commitish = describe[0] 111 | if valid_semver: 112 | version_info = semver.parse_version_info(tag) 113 | if not version_info.prerelease: 114 | version = semver.bump_patch(tag) + "-alpha.0.plus." + additional_commits + "+" + commitish 115 | else: 116 | version = tag + ".plus." + additional_commits + "+" + commitish 117 | else: 118 | version = commitish 119 | return version 120 | 121 | def mpy_cross(version, quiet=False): 122 | circuitpython_tag = version["tag"] 123 | name = version["name"] 124 | ext = ".exe" * (os.name == "nt") 125 | mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}" 126 | 127 | if os.path.isfile(mpy_cross_filename): 128 | return mpy_cross_filename 129 | 130 | # Try to pull from S3 131 | uname = platform.uname() 132 | s3_url = None 133 | if uname[0].title() == 'Linux' and uname[4].lower() in ('amd64', 'x86_64'): 134 | s3_url = f"{S3_MPY_PREFIX}/linux-amd64/mpy-cross-linux-amd64-{circuitpython_tag}.static" 135 | elif uname[0].title() == 'Linux' and uname[4].lower() == 'armv7l': 136 | s3_url = f"{S3_MPY_PREFIX}/linux-raspbian/mpy-cross-linux-raspbian-{circuitpython_tag}.static-raspbian" 137 | elif uname[0].title() == 'Darwin': 138 | s3_url = f"{S3_MPY_PREFIX}/macos-11/mpy-cross-macos-11-{circuitpython_tag}-universal" 139 | elif uname[0].title() == "Windows" and uname[4].lower() in ("amd64", "x86_64"): 140 | s3_url = f"{S3_MPY_PREFIX}/windows/mpy-cross-windows-{circuitpython_tag}.static.exe" 141 | elif not quiet: 142 | print(f"Pre-built mpy-cross not available for sysname='{uname[0]}' release='{uname[2]}' machine='{uname[4]}'.") 143 | 144 | if s3_url is not None: 145 | if not quiet: 146 | print(f"Checking S3 for {s3_url}") 147 | try: 148 | r = requests.get(s3_url) 149 | if r.status_code == 200: 150 | with open(mpy_cross_filename, "wb") as f: 151 | f.write(r.content) 152 | # Set the User Execute bit 153 | os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR) 154 | if not quiet: 155 | print(" FOUND") 156 | return mpy_cross_filename 157 | except Exception as e: 158 | if not quiet: 159 | print(f" exception fetching from S3: {e}") 160 | if not quiet: 161 | print(" NOT FOUND") 162 | 163 | if not quiet: 164 | title = "Building mpy-cross for circuitpython " + circuitpython_tag 165 | print() 166 | print(title) 167 | print("=" * len(title)) 168 | 169 | build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}" 170 | if not os.path.isdir(build_dir): 171 | subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir]) 172 | 173 | subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir) 174 | subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir) 175 | subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross") 176 | subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross") 177 | 178 | mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}" 179 | if not os.path.exists(mpy_built): 180 | mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}" 181 | 182 | shutil.copy(mpy_built, mpy_cross_filename) 183 | return mpy_cross_filename 184 | 185 | def _munge_to_temp(original_path, temp_file, library_version): 186 | with open(original_path, "r", encoding="utf-8") as original_file: 187 | for line in original_file: 188 | line = line.strip("\n") 189 | if line.startswith("__version__"): 190 | line = line.replace("0.0.0-auto.0", library_version) 191 | line = line.replace("0.0.0+auto.0", library_version) 192 | print(line, file=temp_file) 193 | temp_file.flush() 194 | 195 | def get_package_info(library_path, package_folder_prefix): 196 | lib_path = pathlib.Path(library_path) 197 | parent_idx = len(lib_path.parts) 198 | py_files = [] 199 | package_files = [] 200 | package_info = {} 201 | glob_search = [] 202 | for pattern in GLOB_PATTERNS: 203 | glob_search.extend(list(lib_path.rglob(pattern))) 204 | 205 | pyproject_toml = load_pyproject_toml(lib_path) 206 | py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) 207 | packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) 208 | 209 | blocklisted = [name for name in py_modules if name in pyproject_py_modules_blocklist] 210 | 211 | if blocklisted: 212 | print(f"{lib_path}/settings.toml:1: {blocklisted[0]} blocklisted: not using metadata from pyproject.toml") 213 | py_modules = packages = () 214 | 215 | example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") 216 | if sub_path.is_file()] 217 | 218 | if packages and py_modules: 219 | raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages") 220 | 221 | elif packages: 222 | if len(packages) > 1: 223 | raise ValueError("Only a single package is supported") 224 | package_name = packages[0] 225 | #print(f"Using package name from pyproject.toml: {package_name}") 226 | package_info["is_package"] = True 227 | package_info["module_name"] = package_name 228 | package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") 229 | if sub_path.is_file()] 230 | 231 | elif py_modules: 232 | if len(py_modules) > 1: 233 | raise ValueError("Only a single module is supported") 234 | py_module = py_modules[0] 235 | #print(f"Using module name from pyproject.toml: {py_module}") 236 | package_name = py_module 237 | package_info["is_package"] = False 238 | package_info["module_name"] = py_module 239 | py_files = [lib_path / f"{py_module}.py"] 240 | 241 | else: 242 | print(f"{lib_path}: Using legacy autodetection") 243 | package_info["is_package"] = False 244 | for file in glob_search: 245 | if file.parts[parent_idx] != "examples": 246 | if len(file.parts) > parent_idx + 1: 247 | for prefix in package_folder_prefix: 248 | if file.parts[parent_idx].startswith(prefix): 249 | package_info["is_package"] = True 250 | if package_info["is_package"]: 251 | package_files.append(file) 252 | else: 253 | if file.name in IGNORE_PY: 254 | #print("Ignoring:", file.resolve()) 255 | continue 256 | if file.parent == lib_path: 257 | py_files.append(file) 258 | 259 | if package_files: 260 | package_info["module_name"] = package_files[0].relative_to(library_path).parent.name 261 | elif py_files: 262 | package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] 263 | else: 264 | package_info["module_name"] = None 265 | 266 | if len(py_files) > 1: 267 | raise ValueError("Multiple top level py files not allowed. Please put " 268 | "them in a package or combine them into a single file.") 269 | 270 | package_info["package_files"] = package_files 271 | package_info["py_files"] = py_files 272 | package_info["example_files"] = example_files 273 | 274 | try: 275 | package_info["version"] = version_string(library_path, valid_semver=True) 276 | except ValueError as e: 277 | print(library_path + " has version that doesn't follow SemVer (semver.org)") 278 | print(e) 279 | package_info["version"] = version_string(library_path) 280 | 281 | return package_info 282 | 283 | def library(library_path, output_directory, package_folder_prefix, 284 | mpy_cross=None, example_bundle=False): 285 | lib_path = pathlib.Path(library_path) 286 | package_info = get_package_info(library_path, package_folder_prefix) 287 | py_package_files = package_info["package_files"] + package_info["py_files"] 288 | example_files = package_info["example_files"] 289 | module_name = package_info["module_name"] 290 | 291 | for fn in py_package_files: 292 | base_dir = os.path.join(output_directory, 293 | fn.relative_to(library_path).parent) 294 | if not os.path.isdir(base_dir): 295 | os.makedirs(base_dir) 296 | 297 | library_version = package_info['version'] 298 | 299 | if not example_bundle: 300 | for filename in py_package_files: 301 | full_path = os.path.join(library_path, filename) 302 | output_file = output_directory / filename.relative_to(library_path) 303 | if filename.suffix == ".py": 304 | with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file: 305 | temp_file_name = temp_file.name 306 | try: 307 | _munge_to_temp(full_path, temp_file, library_version) 308 | temp_file.close() 309 | if mpy_cross and os.stat(temp_file.name).st_size != 0: 310 | output_file = output_file.with_suffix(".mpy") 311 | mpy_success = subprocess.call([ 312 | mpy_cross, 313 | "-o", output_file, 314 | "-s", str(filename.relative_to(library_path)), 315 | temp_file.name 316 | ]) 317 | if mpy_success != 0: 318 | raise RuntimeError("mpy-cross failed on", full_path) 319 | else: 320 | shutil.copyfile(temp_file_name, output_file) 321 | finally: 322 | os.remove(temp_file_name) 323 | else: 324 | shutil.copyfile(full_path, output_file) 325 | 326 | requirements_files = lib_path.glob("requirements.txt*") 327 | requirements_files = [f for f in requirements_files if f.stat().st_size > 0] 328 | 329 | toml_files = lib_path.glob("pyproject.toml*") 330 | toml_files = [f for f in toml_files if f.stat().st_size > 0] 331 | requirements_files.extend(toml_files) 332 | 333 | if module_name and requirements_files and not example_bundle: 334 | requirements_dir = pathlib.Path(output_directory).parent / "requirements" 335 | if not os.path.isdir(requirements_dir): 336 | os.makedirs(requirements_dir, exist_ok=True) 337 | requirements_subdir = f"{requirements_dir}/{module_name}" 338 | if not os.path.isdir(requirements_subdir): 339 | os.makedirs(requirements_subdir, exist_ok=True) 340 | for filename in requirements_files: 341 | full_path = os.path.join(library_path, filename) 342 | output_file = os.path.join(requirements_subdir, filename.name) 343 | shutil.copyfile(full_path, output_file) 344 | 345 | for filename in example_files: 346 | full_path = os.path.join(library_path, filename) 347 | 348 | relative_filename_parts = list(filename.relative_to(library_path).parts) 349 | relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1]) 350 | final_relative_filename = os.path.join(*relative_filename_parts) 351 | output_file = os.path.join(output_directory.replace("/lib", "/"), 352 | final_relative_filename) 353 | 354 | os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True) 355 | shutil.copyfile(full_path, output_file) 356 | -------------------------------------------------------------------------------- /circuitpython_build_tools/scripts/build_bundles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2016-2017 Scott Shawcroft for Adafruit Industries 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | 25 | import json 26 | import os 27 | import os.path 28 | import re 29 | import shutil 30 | import subprocess 31 | import sys 32 | import zipfile 33 | 34 | import click 35 | 36 | from circuitpython_build_tools import build 37 | from circuitpython_build_tools import target_versions 38 | 39 | if sys.version_info < (3, 8): 40 | import importlib_metadata 41 | else: 42 | import importlib.metadata as importlib_metadata 43 | 44 | BLINKA_LIBRARIES = [ 45 | "adafruit-blinka", 46 | "adafruit-blinka-bleio", 47 | "adafruit-blinka-displayio", 48 | "adafruit-blinka-pyportal", 49 | "adafruit-python-extended-bus", 50 | "numpy", 51 | "pillow", 52 | "pyasn1", 53 | "pyserial", 54 | "scipy", 55 | "spidev", 56 | ] 57 | 58 | def normalize_dist_name(name: str) -> str: 59 | """Return a normalized pip name""" 60 | return name.lower().replace("_", "-") 61 | 62 | def add_file(bundle, src_file, zip_name): 63 | bundle.write(src_file, zip_name) 64 | file_size = os.stat(src_file).st_size 65 | file_sector_size = file_size 66 | if file_size % 512 != 0: 67 | file_sector_size = (file_size // 512 + 1) * 512 68 | print(zip_name, file_size, file_sector_size) 69 | return file_sector_size 70 | 71 | def get_module_name(library_path, remote_name): 72 | """Figure out the module or package name and return it""" 73 | repo = subprocess.run(f'git remote get-url {remote_name}', shell=True, stdout=subprocess.PIPE, cwd=library_path) 74 | repo = repo.stdout.decode("utf-8", errors="ignore").strip().lower() 75 | if repo[-4:] == ".git": 76 | repo = repo[:-4] 77 | module_name = normalize_dist_name(repo.split("/")[-1]) 78 | 79 | # circuitpython org repos are deployed to pypi without "org" in the pypi name 80 | module_name = re.sub(r"^circuitpython-org-", "circuitpython-", module_name) 81 | return module_name, repo 82 | 83 | def get_bundle_requirements(directory, package_list): 84 | """ 85 | Open the requirements.txt if it exists 86 | Remove anything that shouldn't be a requirement like Adafruit_Blinka 87 | Return the list 88 | """ 89 | 90 | pypi_reqs = set() # For multiple bundle dependency 91 | dependencies = set() # For intra-bundle dependency 92 | 93 | path = directory + "/requirements.txt" 94 | if os.path.exists(path): 95 | with open(path, "r") as file: 96 | requirements = file.read() 97 | file.close() 98 | for line in requirements.split("\n"): 99 | line = line.lower().strip() 100 | if line.startswith("#") or line == "": 101 | # skip comments 102 | pass 103 | else: 104 | # Remove any pip version and platform specifiers 105 | original_name = re.split("[<>=~[;]", line)[0].strip() 106 | # Normalize to match the indexes in package_list 107 | line = normalize_dist_name(original_name) 108 | if line in package_list: 109 | dependencies.add(package_list[line]["module_name"]) 110 | elif line not in BLINKA_LIBRARIES: 111 | # add with the exact spelling from requirements.txt 112 | pypi_reqs.add(original_name) 113 | return sorted(dependencies), sorted(pypi_reqs) 114 | 115 | def build_bundle_json(libs, bundle_version, output_filename, package_folder_prefix, remote_name="origin"): 116 | """ 117 | Generate a JSON file of all the libraries in libs 118 | """ 119 | packages = {} 120 | # TODO simplify this 2-step process 121 | # It mostly exists so that get_bundle_requirements has a way to look up 122 | # "pypi name to bundle name" via `package_list[pypi_name]["module_name"]` 123 | # otherwise it's just shuffling info around 124 | for library_path in libs: 125 | package = {} 126 | package_info = build.get_package_info(library_path, package_folder_prefix) 127 | module_name, repo = get_module_name(library_path, remote_name) 128 | if package_info["module_name"] is not None: 129 | package["module_name"] = package_info["module_name"] 130 | package["pypi_name"] = module_name 131 | package["repo"] = repo 132 | package["is_folder"] = package_info["is_package"] 133 | package["version"] = package_info["version"] 134 | package["path"] = "lib/" + package_info["module_name"] 135 | package["library_path"] = library_path 136 | packages[module_name] = package 137 | 138 | library_submodules = {} 139 | for package in packages.values(): 140 | library = {} 141 | library["package"] = package["is_folder"] 142 | library["pypi_name"] = package["pypi_name"] 143 | library["version"] = package["version"] 144 | library["repo"] = package["repo"] 145 | library["path"] = package["path"] 146 | library["dependencies"], library["external_dependencies"] = get_bundle_requirements(package["library_path"], packages) 147 | library_submodules[package["module_name"]] = library 148 | 149 | out_file = open(output_filename, "w") 150 | json.dump(library_submodules, out_file, sort_keys=True) 151 | out_file.close() 152 | 153 | def build_bundle(libs, bundle_version, output_filename, package_folder_prefix, 154 | build_tools_version="devel", mpy_cross=None, example_bundle=False, remote_name="origin"): 155 | build_dir = "build-" + os.path.basename(output_filename) 156 | top_folder = os.path.basename(output_filename).replace(".zip", "") 157 | build_lib_dir = os.path.join(build_dir, top_folder, "lib") 158 | build_example_dir = os.path.join(build_dir, top_folder, "examples") 159 | if os.path.isdir(build_dir): 160 | print("Deleting existing build.") 161 | shutil.rmtree(build_dir) 162 | total_size = 0 163 | if not example_bundle: 164 | os.makedirs(build_lib_dir) 165 | total_size += 512 166 | os.makedirs(build_example_dir) 167 | total_size += 512 168 | 169 | multiple_libs = len(libs) > 1 170 | 171 | success = True 172 | for library_path in libs: 173 | try: 174 | build.library(library_path, build_lib_dir, package_folder_prefix, 175 | mpy_cross=mpy_cross, example_bundle=example_bundle) 176 | except ValueError as e: 177 | print("build.library failure:", library_path) 178 | print(e) 179 | success = False 180 | 181 | print() 182 | print("Generating VERSIONS") 183 | if multiple_libs: 184 | with open(os.path.join(build_dir, top_folder, "VERSIONS.txt"), "w") as f: 185 | f.write(bundle_version + "\r\n") 186 | versions = subprocess.run(f'git submodule --quiet foreach \"git remote get-url {remote_name} && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs)) 187 | if versions.returncode != 0: 188 | print("Failed to generate versions file. Its likely a library hasn't been " 189 | "released yet.") 190 | success = False 191 | 192 | repo = None 193 | for line in versions.stdout.split(b"\n"): 194 | if not line: 195 | continue 196 | if line.startswith(b"ssh://git@"): 197 | repo = b"https://" + line.split(b"@")[1][:-len(".git")] 198 | elif line.startswith(b"git@"): 199 | repo = b"https://github.com/" + line.split(b":")[1][:-len(".git")] 200 | elif line.startswith(b"https:"): 201 | repo = line.strip()[:-len(".git")] 202 | else: 203 | f.write(repo.decode("utf-8", "strict") + "/releases/tag/" + line.strip().decode("utf-8", "strict") + "\r\n") 204 | 205 | if not success: 206 | print("WARNING: some failures above") 207 | sys.exit(2) 208 | 209 | print() 210 | print("Zipping") 211 | 212 | with zipfile.ZipFile(output_filename, 'w', compression=zipfile.ZIP_DEFLATED) as bundle: 213 | build_metadata = {"build-tools-version": build_tools_version} 214 | bundle.comment = json.dumps(build_metadata).encode("utf-8") 215 | if multiple_libs: 216 | total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt")) 217 | for root, dirs, files in os.walk(build_dir): 218 | ziproot = root[len(build_dir + "/"):] 219 | for filename in files: 220 | total_size += add_file(bundle, os.path.join(root, filename), 221 | os.path.join(ziproot, filename.replace("-", "_"))) 222 | 223 | print() 224 | print(total_size, "B", total_size / 1024, "kiB", total_size / 1024 / 1024, "MiB") 225 | print("Bundled in", output_filename) 226 | 227 | def _find_libraries(current_path, depth): 228 | if depth <= 0: 229 | return [current_path] 230 | subdirectories = [] 231 | for subdirectory in os.listdir(current_path): 232 | path = os.path.join(current_path, subdirectory) 233 | if os.path.isdir(path): 234 | subdirectories.extend(_find_libraries(path, depth - 1)) 235 | return subdirectories 236 | 237 | all_modules = ["py", "mpy", "example", "json"] 238 | @click.command() 239 | @click.option('--filename_prefix', required=True, help="Filename prefix for the output zip files.") 240 | @click.option('--output_directory', default="bundles", help="Output location for the zip files.") 241 | @click.option('--library_location', required=True, help="Location of libraries to bundle.") 242 | @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") 243 | @click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.") 244 | @click.option('--remote_name', default="origin", help="Git remote name to use during building") 245 | @click.option('--ignore', "-i", multiple=True, type=click.Choice(all_modules), help="Bundles to ignore building") 246 | @click.option('--only', "-o", multiple=True, type=click.Choice(all_modules), help="Bundles to build building") 247 | def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore, only): 248 | os.makedirs(output_directory, exist_ok=True) 249 | 250 | package_folder_prefix = package_folder_prefix.split(", ") 251 | 252 | bundle_version = build.version_string() 253 | 254 | libs = _find_libraries(os.path.abspath(library_location), library_depth) 255 | 256 | try: 257 | build_tools_version = importlib_metadata.version("circuitpython-build-tools") 258 | except importlib_metadata.PackageNotFoundError: 259 | build_tools_version = "devel" 260 | 261 | build_tools_fn = "z-build_tools_version-{}.ignore".format( 262 | build_tools_version) 263 | build_tools_fn = os.path.join(output_directory, build_tools_fn) 264 | with open(build_tools_fn, "w") as f: 265 | f.write(build_tools_version) 266 | 267 | if ignore and only: 268 | raise SystemExit("Only specify one of --ignore / --only") 269 | if only: 270 | ignore = set(all_modules) - set(only) 271 | 272 | # Build raw source .py bundle 273 | if "py" not in ignore: 274 | zip_filename = os.path.join(output_directory, 275 | filename_prefix + '-py-{VERSION}.zip'.format( 276 | VERSION=bundle_version)) 277 | build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, 278 | build_tools_version=build_tools_version, remote_name=remote_name) 279 | 280 | # Build .mpy bundle(s) 281 | if "mpy" not in ignore: 282 | for version in target_versions.VERSIONS: 283 | mpy_cross = build.mpy_cross(version) 284 | zip_filename = os.path.join(output_directory, 285 | filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( 286 | TAG=version["name"], 287 | VERSION=bundle_version)) 288 | build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, 289 | mpy_cross=mpy_cross, build_tools_version=build_tools_version, remote_name=remote_name) 290 | 291 | # Build example bundle 292 | if "example" not in ignore: 293 | zip_filename = os.path.join(output_directory, 294 | filename_prefix + '-examples-{VERSION}.zip'.format( 295 | VERSION=bundle_version)) 296 | build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, 297 | build_tools_version=build_tools_version, example_bundle=True, remote_name=remote_name) 298 | 299 | # Build Bundle JSON 300 | if "json" not in ignore: 301 | json_filename = os.path.join(output_directory, 302 | filename_prefix + '-{VERSION}.json'.format( 303 | VERSION=bundle_version)) 304 | build_bundle_json(libs, bundle_version, json_filename, package_folder_prefix, remote_name=remote_name) 305 | -------------------------------------------------------------------------------- /circuitpython_build_tools/scripts/build_mpy_cross.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2017 Scott Shawcroft for Adafruit Industries 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | 25 | from circuitpython_build_tools import build 26 | from circuitpython_build_tools import target_versions 27 | 28 | import os 29 | import sys 30 | 31 | import click 32 | 33 | @click.command 34 | @click.argument("versions") 35 | def main(versions): 36 | print(versions) 37 | for version in [v for v in target_versions.VERSIONS if v['name'] in versions]: 38 | print(f"{version['name']}: {build.mpy_cross(version)}") 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /circuitpython_build_tools/scripts/circuitpython_mpy_cross.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | import click 4 | 5 | from ..target_versions import VERSIONS 6 | from ..build import mpy_cross 7 | 8 | @click.command(context_settings={"ignore_unknown_options": True}) 9 | @click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS])) 10 | @click.option("--quiet/--no-quiet", "quiet", type=bool, default=True) 11 | @click.argument("mpy-cross-args", nargs=-1, required=True) 12 | def main(circuitpython_version, quiet, mpy_cross_args): 13 | version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version] 14 | mpy_cross_exe = str(mpy_cross(version_info, quiet)) 15 | try: 16 | subprocess.check_call([mpy_cross_exe, *mpy_cross_args]) 17 | except subprocess.CalledProcessError as e: 18 | raise SystemExit(e.returncode) 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /circuitpython_build_tools/target_versions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | 25 | # The tag specifies which version of CircuitPython to use for mpy-cross. 26 | # The name is used when constructing the zip file names. 27 | VERSIONS = [ 28 | {"tag": "9.2.4", "name": "9.x"}, 29 | {"tag": "10.0.0-alpha.2", "name": "10.x"}, 30 | ] 31 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Click 2 | requests 3 | semver 4 | wheel 5 | tomli; python_version < "3.11" 6 | platformdirs 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup(name='circuitpython-build-tools', 6 | use_scm_version=True, 7 | setup_requires=["setuptools_scm"], 8 | description='CircuitPython library build tools', 9 | author='Scott Shawcroft', 10 | author_email='scott@adafruit.com', 11 | url='https://www.adafruit.com/', 12 | packages=['circuitpython_build_tools', 13 | 'circuitpython_build_tools.scripts'], 14 | package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, 15 | zip_safe=False, 16 | python_requires='>=3.10', 17 | install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'], 18 | entry_points=''' 19 | [console_scripts] 20 | circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles 21 | circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main 22 | ''' 23 | ) 24 | --------------------------------------------------------------------------------