├── .editorconfig ├── .git-blame-ignore-revs ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── publish-pypi.yml │ ├── sync-jira.yml │ ├── test-build-docs.yml │ └── test-build-idf-apps.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── _apidoc_templates │ ├── module.rst_t │ ├── package.rst_t │ └── toc.rst_t ├── _static │ ├── espressif-logo.svg │ └── theme_overrides.css ├── _templates │ └── layout.html ├── conf_common.py └── en │ ├── Makefile │ ├── conf.py │ ├── explanations │ ├── build.rst │ ├── config_rules.rst │ ├── dependency_driven_build.rst │ └── find.rst │ ├── guides │ ├── 1.x_to_2.x.md │ └── custom_app.md │ ├── index.rst │ ├── others │ ├── CHANGELOG.md │ └── CONTRIBUTING.md │ └── references │ ├── cli.rst │ ├── config_file.rst │ └── manifest.rst ├── idf_build_apps ├── __init__.py ├── __main__.py ├── app.py ├── args.py ├── autocompletions.py ├── constants.py ├── finder.py ├── junit │ ├── __init__.py │ ├── report.py │ └── utils.py ├── log.py ├── main.py ├── manifest │ ├── __init__.py │ ├── manifest.py │ └── soc_header.py ├── py.typed ├── session_args.py ├── utils.py ├── vendors │ ├── __init__.py │ └── pydantic_sources.py └── yaml │ ├── __init__.py │ └── parser.py ├── license_header.txt ├── pyproject.toml └── tests ├── conftest.py ├── test_app.py ├── test_args.py ├── test_build.py ├── test_cmd.py ├── test_finder.py ├── test_manifest.py └── test_utils.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | 11 | [*.py] 12 | charset = utf-8 13 | indent_style = space 14 | indent_size = 4 15 | 16 | [*.rst] 17 | indent_style = space 18 | indent_size = 3 19 | 20 | [*.md] 21 | indent_style = space 22 | indent_size = 4 23 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | 49879900d35f6cd34762d9c50aaa8f144d784f96 2 | aeeb64c027a3a0db195749fcba206620e7c9060e 3 | e9d5def7afa2f960da7d7b5863cea95e32898509 4 | 703aec369076479d3d841003f0e193d02df43505 5 | dd8d3634c410c444029e1640e83a5c2f3280be47 6 | f146380f3268c8820d2762155863c3aec40411bc 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | CHANGELOG.md merge=union 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/publish-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish PyPI 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-python@v5 14 | with: 15 | python-version: "3.7" 16 | - name: Publish packages 17 | env: 18 | FLIT_USERNAME: __token__ 19 | FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 20 | run: | 21 | pip install flit 22 | flit publish --setup-py 23 | -------------------------------------------------------------------------------- /.github/workflows/sync-jira.yml: -------------------------------------------------------------------------------- 1 | # FILE: .github/workflows/sync-jira.yml 2 | --- 3 | # This GitHub Actions workflow synchronizes GitHub issues, comments, and pull requests with Jira. 4 | # It triggers on new issues, issue comments, and on a scheduled basis. 5 | # The workflow uses a custom action to perform the synchronization with Jira (espressif/sync-jira-actions). 6 | 7 | name: 🔷 Sync to Jira 8 | 9 | run-name: > 10 | Sync to Jira - 11 | ${{ github.event_name == 'issue_comment' && 'Issue Comment' || 12 | github.event_name == 'schedule' && 'New Pull Requests' || 13 | github.event_name == 'issues' && 'New Issue' || 14 | github.event_name == 'workflow_dispatch' && 'Manual Sync' }} 15 | 16 | on: 17 | issues: { types: [opened] } 18 | issue_comment: { types: [created, edited, deleted] } 19 | schedule: [cron: "0 * * * *"] 20 | workflow_dispatch: 21 | inputs: 22 | action: 23 | { 24 | description: "Action to be performed", 25 | required: true, 26 | default: "mirror-issues", 27 | } 28 | issue-numbers: 29 | { 30 | description: "Issue numbers to sync (comma-separated)", 31 | required: true, 32 | } 33 | 34 | jobs: 35 | sync-to-jira: 36 | name: > 37 | Sync to Jira - 38 | ${{ github.event_name == 'issue_comment' && 'Issue Comment' || 39 | github.event_name == 'schedule' && 'New Pull Requests' || 40 | github.event_name == 'issues' && 'New Issue' || 41 | github.event_name == 'workflow_dispatch' && 'Manual Sync' }} 42 | runs-on: ubuntu-latest 43 | permissions: 44 | contents: read 45 | issues: write 46 | pull-requests: write 47 | steps: 48 | - name: Check out 49 | uses: actions/checkout@v4 50 | 51 | - name: Run synchronization to Jira 52 | uses: espressif/sync-jira-actions@v1 53 | with: 54 | cron_job: ${{ github.event_name == 'schedule' && 'true' || '' }} 55 | env: 56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | JIRA_PASS: ${{ secrets.JIRA_PASS }} 58 | JIRA_PROJECT: RDT 59 | JIRA_COMPONENT: idf-build-apps 60 | JIRA_URL: ${{ secrets.JIRA_URL }} 61 | JIRA_USER: ${{ secrets.JIRA_USER }} 62 | -------------------------------------------------------------------------------- /.github/workflows/test-build-docs.yml: -------------------------------------------------------------------------------- 1 | name: Test Build Docs 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | build-docs: 8 | runs-on: ubuntu-22.04 9 | steps: 10 | - uses: actions/checkout@v4 11 | - name: Set up Python 12 | uses: actions/setup-python@v5 13 | with: 14 | python-version: '3.7' 15 | - name: Install dependencies 16 | run: | 17 | python -m pip install --upgrade pip 18 | pip install flit 19 | flit install -s 20 | - name: Build the docs 21 | run: | 22 | cd docs 23 | pushd en && make html && popd 24 | - name: markdown-link-check 25 | uses: gaurav-nelson/github-action-markdown-link-check@1.0.16 26 | -------------------------------------------------------------------------------- /.github/workflows/test-build-idf-apps.yml: -------------------------------------------------------------------------------- 1 | name: Test Build IDF Apps 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - 'idf_build_apps/**' 7 | push: 8 | branches: 9 | - main 10 | 11 | env: 12 | IDF_PATH: /opt/esp/idf 13 | 14 | defaults: 15 | run: 16 | shell: bash 17 | 18 | jobs: 19 | build-python-packages: 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - uses: actions/checkout@v4 23 | - uses: actions/setup-python@v5 24 | with: 25 | python-version: '3.7' 26 | - run: | 27 | pip install -U pip 28 | pip install flit 29 | flit build 30 | - name: Upload built python packages 31 | uses: actions/upload-artifact@v4 32 | with: 33 | name: wheel 34 | path: dist/idf_build_apps-*.whl 35 | 36 | build-apps-on-idf-env: 37 | if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test-old-idf-releases') }} 38 | needs: build-python-packages 39 | strategy: 40 | matrix: 41 | idf-branch: [ release-v5.0, release-v5.1, release-v5.2, release-v5.3 ] 42 | runs-on: ubuntu-latest 43 | container: 44 | image: espressif/idf:${{ matrix.idf-branch }} 45 | steps: 46 | - name: Download wheel 47 | uses: actions/download-artifact@v4 48 | with: 49 | name: wheel 50 | - name: Build the Apps 51 | run: | 52 | bash $IDF_PATH/install.sh 53 | . $IDF_PATH/export.sh 54 | pip install idf_build_apps-*.whl 55 | cd $IDF_PATH/examples/get-started/hello_world 56 | echo 'CONFIG_IDF_TARGET="esp32"' >sdkconfig.defaults 57 | idf-build-apps build --build-dir build_@t --size-file size_info.json 58 | test -f build_esp32/hello_world.bin 59 | test -f build_esp32/size_info.json 60 | test ! -f build_esp32s2/hello_world.bin 61 | 62 | build-apps-on-idf-master: 63 | runs-on: ubuntu-latest 64 | container: 65 | image: espressif/idf:latest 66 | env: 67 | FLIT_ROOT_INSTALL: 1 68 | steps: 69 | - uses: actions/checkout@v4 70 | - name: Build the Apps 71 | run: | 72 | bash $IDF_PATH/install.sh 73 | . $IDF_PATH/export.sh 74 | pip install flit 75 | flit install -s 76 | python -m idf_build_apps build -vv -t esp32 \ 77 | -p $IDF_PATH/examples/get-started/hello_world \ 78 | --size-file size_info.json 79 | pytest --cov idf_build_apps --cov-report term-missing:skip-covered --junit-xml pytest.xml | tee pytest-coverage.txt 80 | - name: Pytest coverage comment 81 | if: github.event_name == 'pull_request' 82 | uses: MishaKav/pytest-coverage-comment@main 83 | with: 84 | pytest-coverage-path: pytest-coverage.txt 85 | junitxml-path: pytest.xml 86 | 87 | build-apps-on-idf-8266: 88 | runs-on: ubuntu-latest 89 | container: 90 | image: python:3 91 | env: 92 | TOOLCHAIN_DIR: ${HOME}/.espressif/tools 93 | FLIT_ROOT_INSTALL: 1 94 | strategy: 95 | matrix: 96 | branch: 97 | - v3.4 98 | steps: 99 | - uses: actions/checkout@v4 100 | - name: Install dependencies 101 | run: | 102 | apt update \ 103 | && apt install -y --no-install-recommends \ 104 | gcc \ 105 | git \ 106 | wget \ 107 | make \ 108 | libncurses-dev \ 109 | flex \ 110 | bison \ 111 | gperf 112 | - name: Checkout the SDK 113 | run: | 114 | git clone --recursive --shallow-submodules \ 115 | --branch ${{ matrix.branch }} \ 116 | https://github.com/espressif/ESP8266_RTOS_SDK \ 117 | $IDF_PATH 118 | - name: Install toolchain 119 | run: | 120 | ${IDF_PATH}/install.sh 121 | - name: Build Hello World 122 | run: | 123 | . ${IDF_PATH}/export.sh 124 | pip install flit 125 | flit install -s 126 | idf-build-apps build -vv -t esp8266 \ 127 | --build-system make \ 128 | -p ${IDF_PATH}/examples/get-started/hello_world \ 129 | --build-dir build_@t \ 130 | --size-file size_info.json 131 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/*/_build/ 73 | docs/en/references/api/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | .idea/ 162 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: mixed-line-ending 8 | args: [ '-f=lf' ] 9 | - repo: https://github.com/Lucas-C/pre-commit-hooks 10 | rev: v1.5.5 11 | hooks: 12 | - id: insert-license 13 | files: \.py$ 14 | args: 15 | - --license-filepath 16 | - license_header.txt # defaults to: LICENSE.txt 17 | - --use-current-year 18 | exclude: 'idf_build_apps/vendors/' 19 | - repo: https://github.com/astral-sh/ruff-pre-commit 20 | rev: 'v0.11.10' 21 | hooks: 22 | - id: ruff 23 | args: ['--fix'] 24 | - id: ruff-format 25 | - repo: https://github.com/pre-commit/mirrors-mypy 26 | rev: 'v1.15.0' 27 | hooks: 28 | - id: mypy 29 | args: ['--warn-unused-ignores'] 30 | additional_dependencies: 31 | - pydantic 32 | - pydantic-settings 33 | - packaging 34 | - toml 35 | - pyparsing 36 | - types-PyYAML 37 | - types-toml 38 | - pytest 39 | - argcomplete>=3 40 | - esp-bool-parser>=0.1.2,<1 41 | - rich 42 | - repo: https://github.com/hfudev/rstfmt 43 | rev: v0.1.4 44 | hooks: 45 | - id: rstfmt 46 | args: ['-w', '-1'] 47 | files: "docs\/.+rst$" 48 | additional_dependencies: 49 | - sphinx-argparse 50 | - sphinx-tabs 51 | - sphinxcontrib-mermaid 52 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | version: 2 6 | 7 | sphinx: 8 | # Path to your Sphinx configuration file. 9 | configuration: docs/en/conf.py 10 | 11 | build: 12 | os: ubuntu-22.04 13 | tools: 14 | python: "3.7" 15 | 16 | python: 17 | install: 18 | - method: pip 19 | path: . 20 | extra_requirements: 21 | - doc 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## v2.11.2 (2025-06-10) 6 | 7 | ### Fix 8 | 9 | - missing `FolderRule` in .manifest 10 | 11 | ## v2.11.1 (2025-06-04) 12 | 13 | ### Fix 14 | 15 | - stop searching apps when reached file system root 16 | 17 | ## v2.11.0 (2025-06-03) 18 | 19 | ### Feat 20 | 21 | - support extra_pythonpaths injection during the runtime 22 | 23 | ## v2.10.3 (2025-06-03) 24 | 25 | ### Fix 26 | 27 | - app.target have higher precedence than target while `find_apps` 28 | - respect FolderRule.DEFAULT_BUILD_TARGETS while validating app 29 | 30 | ### Refactor 31 | 32 | - move `FolderRule.DEFAULT_BUILD_TARGET` into contextvar 33 | 34 | ## v2.10.2 (2025-05-22) 35 | 36 | ### Perf 37 | 38 | - `most_suitable_rule` stop searching till reached root dir 39 | - pre-compute rules folder, reduced 50% time on `most_suitable_rule` 40 | 41 | ## v2.10.1 (2025-05-05) 42 | 43 | ### Fix 44 | 45 | - cache custom app classes 46 | 47 | ## v2.10.0 (2025-04-22) 48 | 49 | ### Feat 50 | 51 | - support custom class load from CLI 52 | 53 | ## v2.9.0 (2025-04-16) 54 | 55 | ### Feat 56 | 57 | - record manifest_path that introduced the folder rule 58 | - support env var expansion in some fields 59 | 60 | ## v2.8.1 (2025-03-04) 61 | 62 | ### Fix 63 | 64 | - --override-sdkconfig-files not working 65 | 66 | ## v2.8.0 (2025-02-20) 67 | 68 | ### Feat 69 | 70 | - support '--disable-targets' 71 | 72 | ## v2.7.0 (2025-02-18) 73 | 74 | ### Feat 75 | 76 | - improve debug info with rich 77 | 78 | ## v2.6.4 (2025-02-14) 79 | 80 | ### Fix 81 | 82 | - collect file not created when no apps built 83 | 84 | ## v2.6.3 (2025-02-11) 85 | 86 | ### Fix 87 | 88 | - stop returning duplicated apps in `find_apps` 89 | - compare app based on normalized paths 90 | - remove unnecessary check args in dependency-driven-build 91 | 92 | ## v2.6.2 (2025-01-21) 93 | 94 | ### Fix 95 | 96 | - windows root dir returns '\\' instead of the drive 97 | 98 | ## v2.6.1 (2025-01-13) 99 | 100 | ### Fix 101 | 102 | - --config-file not refreshed 103 | 104 | ## v2.6.0 (2025-01-02) 105 | 106 | ### Feat 107 | 108 | - `manifest_rootpath` support env vars expansion 109 | 110 | ### Fix 111 | 112 | - DeprecationWarning: 'count' is passed as positional argument when `re.sub` 113 | - add `py.typed` file to be used in mypy 114 | - negative value for soc caps integer 115 | - **config_file**: recursively load config file for TOML file 116 | 117 | ## v2.5.3 (2024-10-04) 118 | 119 | ### Feat 120 | 121 | - support --manifest-filepatterns 122 | 123 | ## v2.5.2 (2024-09-27) 124 | 125 | ### Fix 126 | 127 | - unset CLI argument wrongly overwrite config file settings with default value 128 | - allow unknown fields 129 | 130 | ## v2.5.1 (2024-09-26) 131 | 132 | ### Fix 133 | 134 | - stop using lambda functions since they cannot be pickled 135 | 136 | ## v2.5.0 (2024-09-26) 137 | 138 | ### Feat 139 | 140 | - raise exception when chaining `or`/`and` in manifest file if statements 141 | - support `idf-build-apps find` with checking modified manifest files 142 | - support `idf-build-apps dump-manifest-sha` 143 | 144 | ### Fix 145 | 146 | - stop calling `sys.exit` when return code is 0 147 | - load config file before cli arguments and func arguments 148 | - pickle dump default protocol different in python 3.7 149 | - loose env var requirements. `IDF_PATH` not required 150 | - stop print build log as error when build failed due to `--warning-as-error` 151 | - requires typing-extensions below 3.11 152 | - stop wrongly created/deleted temporary build log file 153 | 154 | ### Refactor 155 | 156 | - declare argument once. used in both function, cli, and docs 157 | - move Manifest.ROOTPATH to arguments 158 | - expand @p placeholders in `BuildArguments` 159 | 160 | ## v2.4.3 (2024-08-07) 161 | 162 | ### Feat 163 | 164 | - set default building target to "all" if `--target` is not specified 165 | - set default paths to current directory if `--paths` is not specified 166 | 167 | ## v2.4.2 (2024-08-01) 168 | 169 | ### Feat 170 | 171 | - support `--enable-preview-targets` 172 | - support `--include-all-apps` while find_apps 173 | - support `--output-format json` while find_apps 174 | - support `include_disabled_apps` while `find_apps` 175 | 176 | ### Fix 177 | 178 | - ignore specified target if unknown in current ESP-IDF branch instead of raise exception 179 | - correct `post_build` actions for succeeded with warnings builds 180 | - **completions**: fix typos in help 181 | 182 | ### Refactor 183 | 184 | - update deprecated `datetime.utcnow` 185 | 186 | ## v2.4.1 (2024-06-18) 187 | 188 | ### Fix 189 | 190 | - use esp32c5 mp as default path 191 | 192 | ## v2.4.0 (2024-06-17) 193 | 194 | ### Feat 195 | 196 | - support esp32c5 soc header 197 | - **cli**: add CLI autocompletions 198 | 199 | ## v2.3.1 (2024-04-22) 200 | 201 | ### Fix 202 | 203 | - copy sdkconfig file while `_post_build` instead of the final phase of `build_apps` 204 | 205 | ## v2.3.0 (2024-03-20) 206 | 207 | ### Feat 208 | 209 | - support ignore app dependencies by components 210 | 211 | ## v2.2.2 (2024-03-13) 212 | 213 | ### Fix 214 | 215 | - skip size json generation for targets in preview 216 | 217 | ## v2.2.1 (2024-03-04) 218 | 219 | ### Fix 220 | 221 | - override sdkconfig item keep possible double quotes 222 | 223 | ## v2.2.0 (2024-02-22) 224 | 225 | ### Feat 226 | 227 | - Support switch-like statements in `depends_components`, and `depends_filepatterns` 228 | 229 | ## v2.1.1 (2024-02-02) 230 | 231 | ### Fix 232 | 233 | - parse the manifest when folder rule is empty 234 | 235 | ## v2.1.0 (2024-02-01) (yanked) 236 | 237 | ### Feat 238 | 239 | - support postfixes to reuse arrays 240 | 241 | ### Fix 242 | 243 | - wrongly applied to rules which is defined not in parent dir 244 | - same manifest folder rules shouldn't be declared multi times 245 | 246 | ## v2.0.1 (2024-01-15) 247 | 248 | ### Fix 249 | 250 | - wrongly skipped the build when `depends_filepatterns` matched but no component modified 251 | 252 | ## v2.0.0 (2024-01-11) 253 | 254 | ### Feat 255 | 256 | - check if the folders listed in the manifest rules exist or not 257 | - record build status in `App` instance 258 | - support build with `make` 259 | - support `--junitxml` option to generate junitxml report for `build` 260 | - add `AppDeserializer` for differentiating `CMakeApp` and `MakeApp` while deserializing 261 | - add param `check_app_dependencies` in `build_apps` function 262 | - add param `include_skipped_apps` in `find_apps` function 263 | - `find_apps` support custom app class for param `build_system` 264 | - record should_be_built reason when checking app dependencies 265 | - support override sdkconfig CLI Options `--override-sdkconfig-items` and `--override-sdkconfig-files` 266 | - support custom `_pre_build`, `_post_build` in App instances 267 | - add `json_to_app` method with support custom classes 268 | - support `init_from_another` function in App instances 269 | 270 | ### Fix 271 | 272 | - prioritize special rules defined in manifest files 273 | - manifest folder rule starts with a `.` will be skipped checking existence 274 | - log format more visible, from `BUILD_STAGE|` to `[BUILD_STAGE]` 275 | - `app.size_json_path` always returns None for linux target apps 276 | - `app.build_path` returns full path when `build_dir` is a full path, returns relative path otherwise. Before this change, it always returns full path. 277 | - skip build while `find_apps` if `modified_components` is an empty list 278 | - improve error message when env var `IDF_PATH` not set 279 | - correct the search sdkconfig path function 280 | - Turn `app.build()` arguments to kwargs 281 | 282 | ### Changes 283 | 284 | - improve logging output. Differentiate print and logging better. print only when calling this tool via the CLI, not when using as a library 285 | 286 | ### Perf 287 | 288 | - refactor `pathlib` calls to `os.path`, to speed up the function calls 289 | 290 | ### BREAKING CHANGES 291 | 292 | 2.x introduces a lot of breaking changes. For a detailed migration guide, please refer to our [Migration From 1.x to 2.x Guide](https://docs.espressif.com/projects/idf-build-apps/en/latest/guides/1.x_to_2.x.html) 293 | 294 | Here are the breaking changes: 295 | 296 | - make `find_apps`, `build_apps`, keyword-only for most of the params 297 | - migrate `App` class to pydantic model 298 | - update dependencies and do code upgrade to python 3.7 299 | - correct `find_apps`, `build_apps` function params. These files would be generated under the build directory. 300 | - `build_log_path` -> `build_log_filename` 301 | - `size_json_path` -> `size_json_filename` 302 | - differentiate `None` or empty list better while checking, now these params are accepting semicolon-separated list, instead of space-separated list. 303 | - `--modified-components` 304 | - `--modified-files` 305 | - `--ignore-app-dependencies-filepatterns` 306 | - make `App` init function keyword-only for most of the params 307 | - remove `LOGGER` from `idf_build_apps`, use `logging.getLogger('idf_build_apps')` instead 308 | - rename `build_job.py` to `build_apps_args.py`, `BuildAppJob` to `BuildAppsArgs` 309 | 310 | ## v1.1.5 (2024-03-20) 311 | 312 | ### Fix 313 | 314 | - python 2.7 old class 315 | - search sdkconfig path 316 | - improve error message when env var IDF_PATH not set 317 | 318 | ## v1.1.4 (2023-12-29) 319 | 320 | ### Fix 321 | 322 | - stop modifying yaml dict shared by yaml anchors 323 | 324 | ## v1.1.3 (2023-11-13) 325 | 326 | ### Fix 327 | 328 | - pyyaml dependency for python version older than 3.5 329 | - stop recursively copy when work dir in app dir 330 | 331 | ## v1.1.2 (2023-08-16) 332 | 333 | ### Feat 334 | 335 | - improve logging when manifest file is invalid 336 | - skip running "idf.py reconfigure" when modified components is empty 337 | 338 | ## v1.1.1 (2023-08-02) 339 | 340 | ### Fix 341 | 342 | - ignore idf_size.py error 343 | 344 | ## v1.1.0 (2023-07-21) 345 | 346 | ### Feat 347 | 348 | - support esp_rom caps as keywords in the manifest file 349 | 350 | ## v1.0.4 (2023-07-20) 351 | 352 | ### Fix 353 | 354 | - stop overriding supported targets with sdkconfig file defined one for disabled app 355 | 356 | ## v1.0.3 (2023-07-19) 357 | 358 | ### Fix 359 | 360 | - correct final reports with skipped apps and failed built apps 361 | - skip while collecting only when both depend components and files unmatched 362 | 363 | ## v1.0.2 (2023-07-05) 364 | 365 | ### Feat 366 | 367 | - support placeholder "@v" 368 | - Support keyword `IDF_VERSION` in the if statement 369 | 370 | ### Fix 371 | 372 | - non-ascii character 373 | - build failed with warnings even without passing `--check-warnings` 374 | 375 | ## v1.0.1 (2023-06-12) 376 | 377 | ### Fixed 378 | 379 | - glob patterns are matched recursively 380 | 381 | ## v1.0.0 (2023-05-25) 382 | 383 | ### Added 384 | 385 | - Support keyword `depends_filepatterns` in the manifest file 386 | - Support expanding environment variables in the manifest files 387 | 388 | ### BREAKING CHANGES 389 | 390 | - Attributes Renamed 391 | - `App.requires_components` renamed to `App.depends_components` 392 | - `FolderRule.requires_components` renamed to `FolderRule.depends_components` 393 | - Functions Renamed 394 | - `Manifest.requires_components()` renamed to `Manifest.depends_components()` 395 | - Signatures Changed 396 | - `App.build()` 397 | - `App.is_modified()` 398 | - `find_apps()` 399 | - `build_apps()` 400 | - CLI Options Renamed 401 | - `--depends-on-components` renamed to `--modified-components` 402 | - `--depends-on-files` renamed to `--modified-files` 403 | - `--ignore-components-dependencies-file-patterns` renamed to `--ignore-app-dependencies-filepatterns` 404 | - Removed the deprecated CLI call methods, now these options only support space-separated list 405 | - `--exclude` 406 | - `--config` 407 | - `--manifest-file` 408 | - `--ignore-warning-str` 409 | - `--default-build-targets` 410 | 411 | ## v0.6.1 (2023-05-10) 412 | 413 | ### Fixed 414 | 415 | - Add missing dependency `pyyaml`. It's wrongly removed in 0.6.0. 416 | 417 | ## v0.6.0 (2023-05-08) (yanked) 418 | 419 | ### Added 420 | 421 | - Support configuration file with 422 | - `tool.idf-build-apps` section under `pyproject.toml` file 423 | - `.idf_build_apps.toml` file 424 | - Improve help message, include default value, config name, and config type 425 | - Improve help message, add DeprecationWarning to change the CLI call method from "specify multiple times" to "space-separated list" for the following CLI options. (will be removed in 1.0.0) 426 | - `--exclude` 427 | - `--config` 428 | - `--manifest-file` 429 | - `--ignore-warning-str` 430 | - Support placeholder `@p` for parallel index 431 | - Support expand placeholders for CLI options `--collect-app-info` and `--collect-size-info` 432 | - Support new keywords `CONFIG_NAME` in the manifest file 433 | 434 | ### Fixed 435 | 436 | - Fix earlier python version pathlib does not support member function `expanduser` issue 437 | - Remove unused dependency `pyyaml` 438 | 439 | ### Refactored 440 | 441 | - Move `utils.setup_logging()` to `log.setup_logging()` 442 | - Make CLI option `--default-build-targets` from comma-separated list to space-separated list (comma-separated list support will be removed in 1.0.0) 443 | 444 | ## v0.5.2 (2023-04-07) 445 | 446 | ### Fixed 447 | 448 | - Remove empty expanded sdkconfig files folder after build 449 | - Split up expanded sdkconfig files folder for different build 450 | 451 | ## v0.5.1 (2023-04-06) 452 | 453 | ### Fixed 454 | 455 | - Build with expanded sdkconfig file would respect the target-specific one under the original path 456 | 457 | ## v0.5.0 (2023-03-29) 458 | 459 | ### Added 460 | 461 | - Add an executable script `idf-build-apps`. Now this tool could be run via `idf-build-apps build ...` instead of `python -m idf_build_apps build ...` 462 | - Support specify `-DSDKCONFIG_DEFAULTS` for `idf.py build` 463 | - via CLI option `--sdkconfig-defaults` 464 | - via environment variable `SDKCONFIG_DEFAULTS` 465 | 466 | ### Fixed 467 | 468 | - CLI option `-t`, `--target` is required, improve the error message 469 | 470 | ## v0.4.1 (2023-03-15) 471 | 472 | ### Fixed 473 | 474 | - Stop writing `app_info` and `size_info` if the build got skipped 475 | - `IDF_VERSION_MAJOR`, `IDF_VERSION_MINOR`, `IDF_VERSION_PATCH` now are integers 476 | - Skip exclude files while removing build directory if files not exist 477 | - Use log level `INFO` for ignored warnings 478 | - Can't use `and` in if clauses 479 | 480 | ## v0.4.0 (2023-03-09) 481 | 482 | This is the last version to support ESP-IDF v4.1 since it's EOL on Feb. 24th, 2023. 483 | 484 | ### Added 485 | 486 | - Support new keywords `IDF_VERSION_MAJOR`, `IDF_VERSION_MINOR`, `IDF_VERSION_PATCH` in the manifest file 487 | - Support colored output by default in UNIX-like systems 488 | - Add `--no-color` CLI option 489 | - Support ignore check component dependencies based on changed files and specified file patterns 490 | - Add `--ignore-component-dependencies-file-patterns` CLI option 491 | - Add `--depends-on-files` CLI option 492 | 493 | ### Fixed 494 | 495 | - Improve the readability of the generated logs 496 | 497 | ## v0.3.2 (2023-03-08) 498 | 499 | ### Fixed 500 | 501 | - `idf.py reconfigure` without setting `IDF_TARGET` 502 | - wrong log level on "Loading manifest file: ...". Set from `INFO` to `DEBUG` 503 | - wrong log level on "Building app \[ID\]: ...". Set from `DEBUG` to `INFO` 504 | 505 | ## v0.3.1 (2023-02-20) 506 | 507 | ### Fixed 508 | 509 | - Relative path defined in the manifest files depend on the current work path 510 | 511 | Added `manifest_rootpath` argument in `find_apps()`. Will use this value instead as the root folder for calculating absolute path 512 | 513 | ## v0.3.0 (2023-01-10) 514 | 515 | ### Added 516 | 517 | - `find_apps`, `build_apps` support `--depends-on-components`, will only find or build apps that require specified components 518 | - manifest file support `requires_components` 519 | 520 | ### Fixed 521 | 522 | - Wrong `App.verified_targets` when `CONFIG_IDF_TARGET` set in app's `sdkconfig.defaults` file 523 | 524 | ## v0.2.1 (2022-09-02) 525 | 526 | ### Fixed 527 | 528 | - Fix `--format json` incompatible issue for IDF branches earlier than 5.0 529 | - Fix type annotations incompatible issue for python versions earlier than 3.7 530 | - Fix f-string incompatible issue for python versions earlier than 3.7 531 | - Fix unpack dictionary ending comma syntax error for python 3.4 532 | 533 | ## v0.2.0 (2022-08-31) 534 | 535 | ### Added 536 | 537 | - Use `--format json` instead of `--json` with `idf_size.py` 538 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributions Guide 2 | 3 | Hi! We're glad that you're interested in contributing to `idf-build-apps`. This document would guide you through the process of setting up the development environment, running tests, and building documentation. 4 | 5 | ## Supported ESP-IDF Versions 6 | 7 | Here's a table shows the supported ESP-IDF versions and the corresponding Python versions. 8 | 9 | | ESP-IDF Version | ESP-IDF Supported Python Versions | idf-build-apps Releases | 10 | |-----------------|-----------------------------------|-------------------------| 11 | | 4.1 | 2.7, 3.4+ | 1.x | 12 | | 4.2 | 2.7, 3.4+ | 1.x | 13 | | 4.3 | 2.7, 3.4+ | 1.x | 14 | | 4.4 | 2.7, 3.4+ | 1.x | 15 | | 5.0 | 3.7+ | main (2.x) | 16 | | 5.1 | 3.7+ | main (2.x) | 17 | | 5.2 | 3.7+ | main (2.x) | 18 | | 5.3 | 3.8+ | main (2.x) | 19 | | 5.4 | 3.8+ | main (2.x) | 20 | | master (5.5) | 3.9+ | main (2.x) | 21 | 22 | `idf-build-apps` is following the semantic versioning. The major version of `idf-build-apps` is the same as the ESP-IDF version it supports. For example, `idf-build-apps` 1.x supports ESP-IDF 4.x, and `idf-build-apps` 2.x supports ESP-IDF 5.x. 23 | 24 | In order to compatible to all 5.x ESP-IDF versions, please don't forget to keep the code compatible with python 3.7, even it's end of life on 2023-06-05. 25 | 26 | Besides, pydantic dropped 3.7 support since 2.6.0. Don't rely on pydantic 2.6.0+ since we're still supporting python 3.7. 27 | 28 | ## Setup the Dev Environment 29 | 30 | 1. Create virtual environment 31 | 32 | ```shell 33 | python -m venv venv 34 | ``` 35 | 36 | 2. Activate the virtual environment 37 | 38 | ```shell 39 | . ./venv/bin/activate 40 | ``` 41 | 42 | 3. Install [flit][flit] 43 | 44 | We use [flit][flit] to build the package and install the dependencies. 45 | 46 | ```shell 47 | pip install flit 48 | ``` 49 | 50 | 4. Install all dependencies 51 | 52 | All dependencies would be installed, and our package `idf-build-apps` would be installed with editable mode. 53 | 54 | ```shell 55 | flit install -s 56 | ``` 57 | 58 | ## Run Testing 59 | 60 | We use [pytest][pytest] for testing. 61 | 62 | ```shell 63 | pytest 64 | ``` 65 | 66 | ## Build Documentation 67 | 68 | We use [sphinx][sphinx] and [autodoc][autodoc] for generating documentation and API references. Besides, we treat warnings as errors while building the documentation. Please fix them before your commits got merged. 69 | 70 | ```shell 71 | cd docs/en && make html 72 | ``` 73 | 74 | For documentation preview, you may use any browser you prefer. The executable has to be searchable in `PATH`. For example we're using firefox here. 75 | 76 | ```shell 77 | firefox _build/html/index.html 78 | ``` 79 | 80 | [flit]: https://flit.pypa.io/en/stable/index.html 81 | [pytest]: https://docs.pytest.org/en/stable/contents.html 82 | [sphinx]: https://www.sphinx-doc.org/en/master/ 83 | [autodoc]: https://www.sphinx-doc.org/en/master/usage/quickstart.html#autodoc 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # idf-build-apps 2 | 3 | [![Documentation Status](https://readthedocs.com/projects/espressif-idf-build-apps/badge/?version=latest)](https://espressif-docs.readthedocs-hosted.com/projects/idf-build-apps/en/latest/) 4 | [![pypi_package_version](https://img.shields.io/pypi/v/idf-build-apps)](https://pypi.org/project/idf_build_apps/) 5 | [![supported_python_versions](https://img.shields.io/pypi/pyversions/idf-build-apps)](https://pypi.org/project/idf_build_apps/) 6 | 7 | `idf-build-apps` is a tool that helps users find and build [ESP-IDF][esp-idf], and [ESP8266 RTOS][esp8266-rtos] projects in a large scale. 8 | 9 | ## What is an `app`? 10 | 11 | A project using [ESP-IDF][esp-idf] SDK, or [ESP8266 RTOS][esp8266-rtos] SDK typically contains: 12 | 13 | - Build recipe in CMake or Make and the main component with app sources 14 | - (Optional) One or more [sdkconfig][sdkconfig] files 15 | 16 | `app` is the abbreviation for application. An application is a set of binary files that is being built with the specified [sdkconfig][sdkconfig] and the target chip. `idf-build-apps` could build one project into a number of applications according to the matrix of these two parameters. 17 | 18 | ## Installation 19 | 20 | ```shell 21 | pip install idf-build-apps 22 | ``` 23 | 24 | or `pipx` 25 | 26 | ```shell 27 | pipx install idf-build-apps 28 | ``` 29 | 30 | ## Basic Usage 31 | 32 | `idf-build-apps` is a python package that could be used as a library or a CLI tool. 33 | 34 | As a CLI tool, it contains three sub-commands. 35 | 36 | - `find` to find the buildable applications 37 | - `build` to build the found applications 38 | - `completions` to activate autocompletions or print instructions for manual activation 39 | 40 | For detailed explanation to all CLI options, you may run 41 | 42 | ```shell 43 | idf-build-apps -h 44 | idf-build-apps find -h 45 | idf-build-apps build -h 46 | idf-build-apps completions -h 47 | ``` 48 | 49 | As a library, you may check the [API documentation][api-doc] for more information. Overall it provides 50 | 51 | - Two functions, `find_apps` and `build_apps` 52 | - Two classes, `CMakeApp` and `MakeApp` 53 | 54 | ## Quick CLI Example 55 | 56 | To build [ESP-IDF hello world example project][hello-world] with ESP32: 57 | 58 | ```shell 59 | idf-build-apps build -p $IDF_PATH/examples/get-started/hello_world/ --target esp32 60 | ``` 61 | 62 | The binary files will be generated under `$IDF_PATH/examples/get-started/hello_world/build` directory. 63 | 64 | ## Documentation 65 | 66 | For detailed information, please refer to [our documentation site][doc]! 67 | 68 | ## Contributing 69 | 70 | Thanks for your contribution! Please refer to our [Contributing Guide](CONTRIBUTING.md) 71 | 72 | [esp-idf]: https://github.com/espressif/esp-idf 73 | [esp8266-rtos]: https://github.com/espressif/ESP8266_RTOS_SDK 74 | [sdkconfig]: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/kconfig.html 75 | [hello-world]: https://github.com/espressif/esp-idf/tree/master/examples/get-started/hello_world 76 | [supported-targets]: https://github.com/espressif/esp-idf/tree/v5.0#esp-idf-release-and-soc-compatibility 77 | [doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/ 78 | [api-doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/references/api/modules.html 79 | -------------------------------------------------------------------------------- /docs/_apidoc_templates/module.rst_t: -------------------------------------------------------------------------------- 1 | {%- if show_headings %} 2 | {{- [basename, "module"] | join(' ') | e | heading }} 3 | 4 | {% endif -%} 5 | .. automodule:: {{ qualname }} 6 | {%- for option in automodule_options %} 7 | :{{ option }}: 8 | {%- endfor %} 9 | -------------------------------------------------------------------------------- /docs/_apidoc_templates/package.rst_t: -------------------------------------------------------------------------------- 1 | {%- macro automodule(modname, options) -%} 2 | .. automodule:: {{ modname }} 3 | {%- for option in options %} 4 | :{{ option }}: 5 | {%- endfor %} 6 | {%- endmacro %} 7 | 8 | {%- macro toctree(docnames) -%} 9 | .. toctree:: 10 | :maxdepth: {{ maxdepth }} 11 | {% for docname in docnames %} 12 | {{ docname }} 13 | {%- endfor %} 14 | {%- endmacro %} 15 | 16 | {%- if is_namespace %} 17 | {{- [pkgname, "namespace"] | join(" ") | e | heading }} 18 | {% else %} 19 | {{- [pkgname, "package"] | join(" ") | e | heading }} 20 | {% endif %} 21 | 22 | {%- if is_namespace %} 23 | .. py:module:: {{ pkgname }} 24 | {% endif %} 25 | 26 | {%- if modulefirst and not is_namespace %} 27 | {{ automodule(pkgname, automodule_options) }} 28 | {% endif %} 29 | 30 | {%- if subpackages %} 31 | {{ toctree(subpackages) }} 32 | {% endif %} 33 | 34 | {%- if submodules %} 35 | {% if separatemodules %} 36 | {{ toctree(submodules) }} 37 | {% else %} 38 | {%- for submodule in submodules %} 39 | {% if show_headings %} 40 | {{- [submodule, "module"] | join(" ") | e | heading(2) }} 41 | {% endif %} 42 | {{ automodule(submodule, automodule_options) }} 43 | {% endfor %} 44 | {%- endif %} 45 | {%- endif %} 46 | 47 | {%- if not modulefirst and not is_namespace %} 48 | {{ automodule(pkgname, automodule_options) }} 49 | {% endif %} 50 | -------------------------------------------------------------------------------- /docs/_apidoc_templates/toc.rst_t: -------------------------------------------------------------------------------- 1 | {{ header | heading }} 2 | 3 | .. toctree:: 4 | :maxdepth: {{ maxdepth }} 5 | {% for docname in docnames %} 6 | {{ docname }} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /docs/_static/espressif-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 12 | 13 | 14 | 16 | 18 | 23 | 26 | 27 | 28 | 33 | 38 | 39 | 42 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 54 | 55 | 62 | 69 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /docs/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | /* override table width restrictions */ 2 | @media screen and (min-width: 767px) { 3 | 4 | .wy-table-responsive table td { 5 | /* !important prevents the common CSS stylesheets from overriding 6 | this as on RTD they are loaded after this stylesheet */ 7 | white-space: normal !important; 8 | } 9 | 10 | .wy-table-responsive { 11 | overflow: visible !important; 12 | } 13 | } 14 | 15 | .wy-side-nav-search { 16 | background-color: #e3e3e3 !important; 17 | } 18 | 19 | .wy-side-nav-search input[type=text] { 20 | border-radius: 0px !important; 21 | border-color: #333333 !important; 22 | } 23 | 24 | .icon-home { 25 | color: #333333 !important; 26 | } 27 | 28 | .icon-home:hover { 29 | background-color: #d6d6d6 !important; 30 | } 31 | 32 | .version { 33 | color: #000000 !important; 34 | } 35 | 36 | a:hover { 37 | color: #bd2c2a !important; 38 | } 39 | 40 | .logo { 41 | width: 240px !important; 42 | } 43 | 44 | .highlight .c1 { 45 | color: #008080; 46 | } 47 | 48 | .bolditalics { 49 | font-weight: bold; 50 | font-style: italic; 51 | } 52 | 53 | pre { 54 | white-space: pre-wrap !important; 55 | } 56 | -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {% block document %} 4 | 5 | {{ super() }} 6 | 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /docs/conf_common.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import shutil 6 | import subprocess 7 | 8 | # Configuration file for the Sphinx documentation builder. 9 | # 10 | # For the full list of built-in configuration values, see the documentation: 11 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 12 | from datetime import datetime 13 | 14 | # -- Project information ----------------------------------------------------- 15 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 16 | 17 | project = 'idf-build-apps' 18 | project_homepage = 'https://github.com/espressif/idf-build-apps' 19 | copyright = f'2023-{datetime.now().year}, Espressif Systems (Shanghai) Co., Ltd.' # noqa: A001 20 | author = 'Fu Hanxi' 21 | languages = ['en'] 22 | version = '2.x' 23 | 24 | # -- General configuration --------------------------------------------------- 25 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 26 | 27 | extensions = [ 28 | 'sphinx.ext.autodoc', 29 | 'sphinx_copybutton', 30 | 'myst_parser', 31 | 'sphinxcontrib.mermaid', 32 | 'sphinxarg.ext', 33 | 'sphinx_tabs.tabs', 34 | ] 35 | 36 | templates_path = ['_templates'] 37 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 38 | 39 | # -- Options for HTML output ------------------------------------------------- 40 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 41 | 42 | html_css_files = ['theme_overrides.css'] 43 | html_logo = '../_static/espressif-logo.svg' 44 | html_static_path = ['../_static'] 45 | html_theme = 'sphinx_rtd_theme' 46 | 47 | # mermaid 10.2.0 will show syntax error 48 | # use fixed version instead 49 | mermaid_version = '10.6.1' 50 | 51 | autodoc_default_options = { 52 | 'members': True, 53 | 'member-order': 'bysource', 54 | 'show-inheritance': True, 55 | 'exclude-members': 'model_computed_fields,model_config,model_fields,model_post_init', 56 | } 57 | 58 | 59 | def generate_api_docs(language): 60 | from idf_build_apps.args import ( 61 | BuildArguments, 62 | FindArguments, 63 | add_args_to_obj_doc_as_params, 64 | ) 65 | from idf_build_apps.main import build_apps, find_apps 66 | 67 | docs_dir = os.path.dirname(__file__) 68 | api_dir = os.path.join(docs_dir, language, 'references', 'api') 69 | if os.path.isdir(api_dir): 70 | shutil.rmtree(api_dir) 71 | 72 | # --- MOCK DOCSTRINGS By Arguments --- 73 | add_args_to_obj_doc_as_params(FindArguments) 74 | add_args_to_obj_doc_as_params(BuildArguments) 75 | add_args_to_obj_doc_as_params(FindArguments, find_apps) 76 | add_args_to_obj_doc_as_params(BuildArguments, build_apps) 77 | # --- MOCK DOCSTRINGS FINISHED --- 78 | 79 | subprocess.run( 80 | [ 81 | 'sphinx-apidoc', 82 | os.path.join(docs_dir, '..', 'idf_build_apps'), 83 | '-f', 84 | '-H', 85 | 'API Reference', 86 | '--no-headings', 87 | '-t', 88 | '_apidoc_templates', 89 | '-o', 90 | api_dir, 91 | ] 92 | ) 93 | -------------------------------------------------------------------------------- /docs/en/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= -W 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/en/conf.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import sys 6 | 7 | language = 'en' 8 | 9 | sys.path.insert(0, os.path.abspath('../')) 10 | 11 | from conf_common import * # noqa 12 | 13 | generate_api_docs(language) # noqa 14 | -------------------------------------------------------------------------------- /docs/en/explanations/build.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Build Command 3 | ############### 4 | 5 | .. note:: 6 | 7 | If you are unfamiliar with the the find command yet, please read the :doc:`find` first. 8 | 9 | This page explains the process of build apps, and how to use the ``idf-build-apps build`` command to build apps in the projects. 10 | 11 | .. note:: 12 | 13 | For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.FindArguments` reference. 14 | 15 | ************************* 16 | Basic ``build`` Command 17 | ************************* 18 | 19 | .. code:: shell 20 | 21 | idf-build-apps build 22 | 23 | ***************************** 24 | ``build`` With Placeholders 25 | ***************************** 26 | 27 | Besides of the :ref:`placeholders supported in find command `, the build command also supports the following placeholders: 28 | 29 | - ``@i``: Would be replaced by the build index. 30 | - ``@p``: Would be replaced by the parallel build index. 31 | 32 | ******************************* 33 | ``build`` With Warnings Check 34 | ******************************* 35 | 36 | You may use `--check-warnings` to enable this check. If any warning is captured while the building process, the exit code would turn to a non-zero value. Besides, `idf-build-apps` provides CLI options `--ignore-warnings-str` and `--ignore-warnings-file` to let you bypass some false alarms. 37 | 38 | *************************** 39 | ``build`` With Debug Mode 40 | *************************** 41 | 42 | It's useful to call `--dry-run` with verbose mode `-vv` to know the whole build process in detail before the build actually happens. For example: 43 | 44 | .. code:: shell 45 | 46 | idf-build-apps build -p . --recursive --target esp32 --dry-run -vv --config "sdkconfig.ci.*=" 47 | 48 | The output would be: 49 | 50 | .. code:: text 51 | 52 | 2024-08-12 15:48:01 DEBUG Looking for CMakeApp apps in . recursively with target esp32 53 | 2024-08-12 15:48:01 DEBUG Entering . 54 | 2024-08-12 15:48:01 DEBUG Skipping. . is not an app 55 | 2024-08-12 15:48:01 DEBUG Entering ./test-1 56 | 2024-08-12 15:48:01 DEBUG sdkconfig file sdkconfig.defaults not found, checking under app_dir... 57 | 2024-08-12 15:48:01 DEBUG Use sdkconfig file ./test-1/sdkconfig.defaults 58 | 2024-08-12 15:48:01 DEBUG Use sdkconfig file /tmp/test/examples/test-1/sdkconfig.ci.bar 59 | 2024-08-12 15:48:01 DEBUG Found app: (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in ./test-1/build 60 | 2024-08-12 15:48:01 DEBUG 61 | 2024-08-12 15:48:01 DEBUG sdkconfig file sdkconfig.defaults not found, checking under app_dir... 62 | 2024-08-12 15:48:01 DEBUG Use sdkconfig file ./test-1/sdkconfig.defaults 63 | 2024-08-12 15:48:01 DEBUG Use sdkconfig file /tmp/test/examples/test-1/sdkconfig.ci.foo 64 | 2024-08-12 15:48:01 DEBUG Found app: (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in ./test-1/build 65 | 2024-08-12 15:48:01 DEBUG 66 | 2024-08-12 15:48:01 DEBUG => Stop iteration sub dirs of ./test-1 since it has apps 67 | 2024-08-12 15:48:01 DEBUG Entering ./test-2 68 | 2024-08-12 15:48:01 DEBUG sdkconfig file sdkconfig.defaults not found, checking under app_dir... 69 | 2024-08-12 15:48:01 DEBUG sdkconfig file ./test-2/sdkconfig.defaults not found, skipping... 70 | 2024-08-12 15:48:01 DEBUG Found app: (cmake) App ./test-2, target esp32, sdkconfig (default), build in ./test-2/build 71 | 2024-08-12 15:48:01 DEBUG 72 | 2024-08-12 15:48:01 DEBUG => Stop iteration sub dirs of ./test-2 since it has apps 73 | 2024-08-12 15:48:01 INFO Found 3 apps in total 74 | 2024-08-12 15:48:01 INFO Total 3 apps. running build for app 1-3 75 | 2024-08-12 15:48:01 INFO (1/3) Building app: (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in ./test-1/build 76 | 2024-08-12 15:48:01 INFO [ Dry Run] Writing build log to ./test-1/build/.temp.build.-4727026790408965348.log 77 | 2024-08-12 15:48:01 INFO skipped (dry run) 78 | 2024-08-12 15:48:01 INFO 79 | 2024-08-12 15:48:01 INFO (2/3) Building app: (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in ./test-1/build 80 | 2024-08-12 15:48:01 INFO [ Dry Run] Writing build log to ./test-1/build/.temp.build.4508471977171905517.log 81 | 2024-08-12 15:48:01 INFO skipped (dry run) 82 | 2024-08-12 15:48:01 INFO 83 | 2024-08-12 15:48:01 INFO (3/3) Building app: (cmake) App ./test-2, target esp32, sdkconfig (default), build in ./test-2/build 84 | 2024-08-12 15:48:01 INFO [ Dry Run] Writing build log to ./test-2/build/.temp.build.4188038822526638365.log 85 | 2024-08-12 15:48:01 INFO skipped (dry run) 86 | 2024-08-12 15:48:01 INFO 87 | Skipped building the following apps: 88 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in ./test-1/build, skipped in 0.000635s: dry run 89 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in ./test-1/build, skipped in 0.000309s: dry run 90 | (cmake) App ./test-2, target esp32, sdkconfig (default), build in ./test-2/build, skipped in 0.000265s: dry run 91 | -------------------------------------------------------------------------------- /docs/en/explanations/config_rules.rst: -------------------------------------------------------------------------------- 1 | ########################## 2 | Sdkconfig & Config Rules 3 | ########################## 4 | 5 | This page explains the concept of `Sdkconfig Files`_ and `Config Rules`_. All examples are based on the following demo project, with the folder structure: 6 | 7 | .. code:: text 8 | 9 | test-1/ 10 | ├── CMakeLists.txt 11 | ├── main/ 12 | │ ├── CMakeLists.txt 13 | │ └── test-1.c 14 | ├── sdkconfig.ci.bar 15 | ├── sdkconfig.ci.foo 16 | ├── sdkconfig.ci.foo.esp32 17 | ├── sdkconfig.defaults 18 | └── sdkconfig.defaults.esp32 19 | 20 | ***************** 21 | Sdkconfig Files 22 | ***************** 23 | 24 | In general, `sdkconfig files`_ are text files that contains the Kconfig items in the form of ``key=value`` pairs. 25 | 26 | Specifically, in ESP-IDF, the ``sdkconfig`` file is the file that contains all the Kconfig items used by the ESP-IDF build system to configure the build process. The ``sdkconfig`` file is generated by the ESP-IDF build system based on the default Kconfig settings, and the `pre-set configurations`_ values. 27 | 28 | Pre-set Configurations 29 | ====================== 30 | 31 | `Pre-set configurations`_ are the Kconfig items that are set in `sdkconfig files`_ before the real build process starts. 32 | 33 | By default, ESP-IDF uses the ``sdkconfig.defaults`` file to set the pre-set configurations. 34 | 35 | In ESP-IDF, the target-specific sdkconfig files are used to override the pre-set configurations, when building the project for a specific target. Target-specific sdkconfig files are always endswith the target name. For example, when building the project for the ESP32 target, the ESP-IDF build system will consider the `sdkconfig files`_ with the ``.esp32`` suffix. The values in these target-specific sdkconfig files will override the pre-set values in the ``sdkconfig.defaults`` file. The `override order`_ is explained in more detail in a later section. 36 | 37 | .. _config-rules: 38 | 39 | ************** 40 | Config Rules 41 | ************** 42 | 43 | In most common CI workflows, the project is built with different configurations. Different combinations of configuration options are tested to ensure the project's robustness. 44 | 45 | The `idf-build-apps` tool uses the concept of `config rules`_ to define the way how the project is built with different pre-set configurations. Each matched `sdkconfig file <#sdkconfig-files>`_ will be built into a separate application, which can be used for testing individually at a later stage. 46 | 47 | Definition 48 | ========== 49 | 50 | To define a Config Rule, use the following format: ``[SDKCONFIG_FILEPATTERN]=[CONFIG_NAME]``. 51 | 52 | - ``SDKCONFIG_FILEPATTERN``: This can be a file name to match a `sdkconfig file <#sdkconfig-files>`_ or a pattern with one wildcard (``*``) character to match multiple `sdkconfig files`_. 53 | - ``CONFIG_NAME``: The name of the corresponding build configuration. This value can be skipped if the wildcard value is to be used. 54 | 55 | The config rules and the corresponding matched `sdkconfig files`_ for the example project are as follows: 56 | 57 | .. list-table:: Config Rules 58 | :widths: 15 15 55 15 59 | :header-rows: 1 60 | 61 | - - Config Rule 62 | - Config Name 63 | - Explanation 64 | - Matched sdkconfig file 65 | 66 | - - ``=`` 67 | - ``default`` 68 | - The default value of the config name is ``default``. 69 | - 70 | 71 | - - ``sdkconfig.ci.foo=test`` 72 | - ``test`` 73 | - 74 | - ``sdkconfig.ci.foo`` 75 | 76 | - - ``sdkconfig.not_exists=test`` 77 | - ``default`` 78 | - The config rule doesn't match any sdkconfig file. The default value is used instead. 79 | - 80 | 81 | - - ``sdkconfig.ci.*=`` 82 | - - ``foo`` 83 | - ``bar`` 84 | - The wildcard matches two files. Two apps are built based on each sdkconfig file. 85 | - - ``sdkconfig.ci.foo`` 86 | - ``sdkconfig.ci.bar`` 87 | 88 | **************** 89 | Override Order 90 | **************** 91 | 92 | The override order is explained using graphs to make it easier to understand. 93 | 94 | Basic ``sdkconfig.defaults`` 95 | ============================ 96 | 97 | The matched `sdkconfig files`_ override the pre-set configurations in the ``sdkconfig.defaults`` file. The priority order is as follows (the arrow points to the higher priority): 98 | 99 | .. mermaid:: 100 | 101 | graph TD 102 | A[sdkconfig.defaults] 103 | B[sdkconfig.ci.foo] 104 | C[sdkconfig.ci.bar] 105 | D{{app foo}} 106 | E{{app bar}} 107 | 108 | subgraph pre-set configurations 109 | A 110 | B 111 | C 112 | end 113 | 114 | A --> B -- "populates sdkconfig file, then build" --> D 115 | A --> C -- "populates sdkconfig file, then build" --> E 116 | 117 | Target-specific Sdkconfig Files 118 | =============================== 119 | 120 | When building the project for the ESP32 target, `sdkconfig files`_ with the ``.esp32`` suffix are considered in addition to the following order (the arrow points to the higher priority): 121 | 122 | .. mermaid:: 123 | 124 | graph TD 125 | A[sdkconfig.defaults] 126 | B[sdkconfig.defaults.esp32] 127 | C[sdkconfig.ci.foo] 128 | D[sdkconfig.ci.foo.esp32] 129 | E[sdkconfig.ci.bar] 130 | F{{app foo}} 131 | G{{app bar}} 132 | 133 | subgraph pre-set configurations 134 | 135 | subgraph only apply when building esp32 136 | B 137 | D 138 | end 139 | 140 | A 141 | C 142 | E 143 | end 144 | 145 | A --> B 146 | B --> C --> D -- "populates sdkconfig file, then build" --> F 147 | B --> E -- "populates sdkconfig file, then build" --> G 148 | 149 | .. warning:: 150 | 151 | Standalone target-specific sdkconfig files are ignored. To make the target-specific sdkconfig files effective, the original sdkconfig file, (without the target name suffix) must be present. 152 | 153 | For example, ``sdkconfig.ci.foo.esp32`` will only be taken into account while building with target ``esp32`` if ``sdkconfig.ci.foo`` is also present. 154 | 155 | Override In CLI 156 | =============== 157 | 158 | ``idf-build-apps`` also supports overriding the pre-set configurations using CLI options. 159 | 160 | - ``--override-sdkconfig-items`` 161 | 162 | A comma-separated list of key-value pairs representing the configuration options. 163 | 164 | - ``--override-sdkconfig-files`` 165 | 166 | A comma-separated list of file paths pointing to the `sdkconfig files`_. 167 | 168 | To make the example more complex, assume that the following CLI options are used: 169 | 170 | - ``--override-sdkconfig-items=CONFIG1=VALUE1,CONFIG2=VALUE2`` 171 | - ``--override-sdkconfig-files=temp1,temp2`` 172 | 173 | Now the priority order of the configuration options is as follows (the arrow points to the higher priority): 174 | 175 | .. mermaid:: 176 | 177 | graph TD 178 | A[sdkconfig.defaults] 179 | B[sdkconfig.defaults.esp32] 180 | C[sdkconfig.ci.foo] 181 | D[sdkconfig.ci.foo.esp32] 182 | E[sdkconfig.ci.bar] 183 | F[temp1] 184 | G[temp2] 185 | H[A temp file, that contains the value of --override-sdkconfig-items 186 | CONFIG1=VALUE1 187 | CONFIG2=VALUE2] 188 | 189 | I{{app foo}} 190 | J{{app bar}} 191 | 192 | subgraph pre-set configurations 193 | 194 | subgraph only apply when building esp32 195 | B 196 | D 197 | end 198 | 199 | A 200 | C 201 | E 202 | F 203 | G 204 | H 205 | end 206 | 207 | A --> B 208 | B --> C --> D --> F --> G --> H -- "populates sdkconfig file, then build" --> I 209 | B --> E --> F --> G --> H -- "populates sdkconfig file, then build" --> J 210 | -------------------------------------------------------------------------------- /docs/en/explanations/dependency_driven_build.rst: -------------------------------------------------------------------------------- 1 | ######################### 2 | Dependency-Driven Build 3 | ######################### 4 | 5 | In large projects or monorepos, it is often desirable to only run builds and tests which are somehow related to the changes in a pull request. 6 | 7 | idf-build-apps supports this by checking whether a particular app has been modified, or depends on modified components or modified files. This check is based on the knowledge of two things: the list of components/files the app depends on, and the list of components/app which are modified in the pull request. 8 | 9 | .. note:: 10 | 11 | For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.DependencyDrivenBuildArguments` reference. 12 | 13 | .. _basic-usage: 14 | 15 | ************* 16 | Basic Usage 17 | ************* 18 | 19 | To enable this feature, the simplest way is to pass ``--modified-components`` to the ``idf-build-apps build`` command. 20 | 21 | While building the app, ``idf-build-apps`` will first run ``idf.py reconfigure``. ``idf.py reconfigure`` will run the first-step of the build system, which will determine the list of components the app depends on. Then, ``idf-build-apps`` will compare the list of modified components with the list of components the app depends on. If any of the modified components are present in the list of dependencies, the app will be built. 22 | 23 | For example, if we run 24 | 25 | .. code:: bash 26 | 27 | cd $IDF_PATH/examples/get-started/hello_world 28 | idf-build-apps build -t esp32 --modified-components fake 29 | 30 | We'll see the following output: 31 | 32 | .. code:: text 33 | 34 | (cmake) App ., target esp32, sdkconfig (default), build in ./build, skipped in 4.271822s: app . depends components: {'esp_app_format', 'esp_driver_sdmmc', 'esp_driver_gpio', 'esp_common', 'esp_driver_parlio', 'esp_http_client', 'esp-tls', 'heap', 'app_trace', 'esp_driver_rmt', 'bt', 'esp_driver_ana_cmpr', 'esptool_py', 'wear_levelling', 'esp_driver_ppa', 'esp_driver_cam', 'unity', 'usb', 'app_update', 'esp_driver_spi', 'protocomm', 'esp_ringbuf', 'esp_security', 'bootloader', 'freertos', 'idf_test', 'vfs', 'hal', 'log', 'nvs_flash', 'esp_system', 'esp_driver_sdio', 'rt', 'efuse', 'esp_https_ota', 'espcoredump', 'esp_timer', 'esp_adc', 'esp_local_ctrl', 'xtensa', 'nvs_sec_provider', 'esp_pm', 'esp_gdbstub', 'lwip', 'json', 'partition_table', 'ulp', 'mbedtls', 'wifi_provisioning', 'esp_driver_sdspi', 'esp_vfs_console', 'esp_partition', 'soc', 'esp_psram', 'esp_eth', 'perfmon', 'sdmmc', 'esp_driver_usb_serial_jtag', 'esp_driver_dac', 'esp_driver_jpeg', 'esp_lcd', 'esp_driver_i2s', 'esp_driver_pcnt', 'ieee802154', 'esp_driver_i2c', 'spiffs', 'esp_driver_tsens', 'driver', 'mqtt', 'main', 'tcp_transport', 'newlib', 'openthread', 'esp_hid', 'esp_driver_gptimer', 'fatfs', 'protobuf-c', 'esp_netif', 'esp_rom', 'cxx', 'esp_bootloader_format', 'esp_wifi', 'esp_driver_ledc', 'pthread', 'esp_phy', 'esp_driver_touch_sens', 'http_parser', 'esp_https_server', 'bootloader_support', 'esp_hw_support', 'esp_event', 'esp_driver_uart', 'esp_netif_stack', 'cmock', 'spi_flash', 'esp_driver_sdm', 'esp_coex', 'esp_driver_isp', 'esp_mm', 'esp_driver_mcpwm', 'wpa_supplicant', 'esp_http_server', 'console'}, while current build modified components: ['fake'] 35 | 36 | The app is skipped because it does not depend on the modified component `fake`. 37 | 38 | ************************************ 39 | Customize the Dependency of an App 40 | ************************************ 41 | 42 | .. note:: 43 | 44 | If you're unfamiliar with the manifest file, please refer to the :doc:`Manifest File Reference <../references/manifest>`. 45 | 46 | To customize the dependencies of an app, `idf-build-apps` supports declaring the dependencies in the manifest files with the `depends_components` and `depends_filepatterns` fields. ``idf-build-apps`` will build the app in the following conditions: 47 | 48 | - any of the files under the app directory are modified, except for the ``.md`` files. 49 | - any of the modified components are listed in the ``depends_components`` field. (if ``depends_components`` specified) 50 | - any of the modified components are listed in the ``idf.py reconfigure`` output. (if ``depends_components`` not specified, as explained in the :ref:`previous section `) 51 | - any of the modified files are matched by the ``depends_filepatterns`` field. 52 | 53 | Here is an example of a manifest file: 54 | 55 | .. code:: yaml 56 | 57 | # rules.yml 58 | examples/foo: 59 | depends_components: 60 | - comp1 61 | - comp2 62 | - comp3 63 | depends_filepatterns: 64 | - "common_header_files/**/*" 65 | 66 | The apps under folder ``examples/foo`` will be built with the following CLI options: 67 | 68 | - ``--manifest-files rules.yml --modified-files examples/foo/main/foo.c`` 69 | 70 | modified file is under the app directory 71 | 72 | - ``--manifest-files rules.yml --modified-components comp1`` 73 | 74 | modified component is listed in the ``depends_components`` field 75 | 76 | - ``--manifest-files rules.yml --modified-components comp2;comp4 --modified-files /tmp/foo.h`` 77 | 78 | modified component is listed in the ``depends_components`` field 79 | 80 | - ``--manifest-files rules.yml --modified-files common_header_files/foo.h`` 81 | 82 | modified file is matched by the ``depends_filepatterns`` field 83 | 84 | - ``--manifest-files rules.yml --modified-components comp4 --modified-files common_header_files/foo.h`` 85 | 86 | modified file is matched by the ``depends_filepatterns`` field 87 | 88 | The apps will not be built with the following CLI options: 89 | 90 | - ``--manifest-files rules.yml --modified-files examples/foo/main/foo.md`` 91 | 92 | only the ``.md`` files are modified 93 | 94 | - ``--manifest-files rules.yml --modified-components bar`` 95 | 96 | modified component is not listed in the ``depends_components`` field 97 | 98 | - ``--modified-components comp1`` 99 | 100 | ``--manifest-files`` is not passed 101 | 102 | The entries in the manifest files are relative paths. By default they are relative to the current working directory. If you want to set the root directory of the manifest files, you can use the ``--manifest-rootpath`` CLI option. 103 | 104 | ********************************************************** 105 | Disable the Feature When Touching Low-level Dependencies 106 | ********************************************************** 107 | 108 | Low-level dependencies, are components or files that are used by many others. For example, component ``freertos`` provides the operating system support for all apps, and ESP-IDF build system related cmake files are also used by all apps. When these items are modified, we definitely need to build and test all the apps. 109 | 110 | To disable the dependency-driven build feature, you can use the CLI option ``--deactivate-dependency-driven-build-by-components`` or ``--deactivate-dependency-driven-build-by-filepatterns``. For example: 111 | 112 | .. code:: bash 113 | 114 | idf-build-apps build -t esp32 --modified-components freertos --deactivate-dependency-driven-build-by-components freertos 115 | 116 | This command will build all the apps, even if the apps do not depend on the component ``freertos``. 117 | -------------------------------------------------------------------------------- /docs/en/explanations/find.rst: -------------------------------------------------------------------------------- 1 | ############## 2 | Find Command 3 | ############## 4 | 5 | .. note:: 6 | 7 | If you are unfamiliar with the concept of sdkconfig files and config rules yet, please read the :doc:`config_rules` first. 8 | 9 | This page explains the process of find apps, and how to use the ``idf-build-apps find`` command to search for apps in the project. 10 | 11 | All examples are based on the following demo projects, with the folder structure: 12 | 13 | .. code:: text 14 | 15 | /tmp/test/examples 16 | ├── test-1 17 | │ ├── CMakeLists.txt 18 | │ ├── main 19 | │ │ ├── CMakeLists.txt 20 | │ │ └── test-1.c 21 | │ ├── sdkconfig.ci.bar 22 | │ ├── sdkconfig.ci.foo 23 | │ ├── sdkconfig.defaults 24 | │ └── sdkconfig.defaults.esp32 25 | └── test-2 26 | ├── CMakeLists.txt 27 | └── main 28 | ├── CMakeLists.txt 29 | └── test-2.c 30 | 31 | .. note:: 32 | 33 | For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.FindArguments` reference. 34 | 35 | ************************ 36 | Basic ``find`` Command 37 | ************************ 38 | 39 | The basic command to find all the buildable apps under ``/tmp/test/examples`` recursively with target ``esp32`` is: 40 | 41 | .. code:: shell 42 | 43 | cd /tmp/test/examples 44 | idf-build-apps find --path . --target esp32 --recursive 45 | 46 | The output would be: 47 | 48 | .. code:: shell 49 | 50 | (cmake) App ./test-1, target esp32, sdkconfig (default), build in ./test-1/build 51 | (cmake) App ./test-2, target esp32, sdkconfig (default), build in ./test-2/build 52 | 53 | The default value of ``--path`` is the current directory, so the ``--path .`` can be omitted. 54 | 55 | .. note:: 56 | 57 | You may check the default values by running ``idf-build-apps find --help`` or check the :doc:`../references/cli`. 58 | 59 | **************************** 60 | ``find`` With Config Rules 61 | **************************** 62 | 63 | To build one project with different configurations, you can use the :ref:`config-rules` to define the build configurations. The ``find`` command will build the project with all the matched configurations. 64 | 65 | For example, 66 | 67 | .. code:: shell 68 | 69 | idf-build-apps find -p test-1 --target esp32 --config "sdkconfig.ci.*=" 70 | 71 | The output would be: 72 | 73 | .. code:: text 74 | 75 | (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in test-1/build 76 | (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in test-1/build 77 | 78 | You may also use :ref:`config-rules` for multiple values: 79 | 80 | .. code:: shell 81 | 82 | idf-build-apps find -p test-1 --target esp32 --config "sdkconfig.ci.*=" "sdkconfig.defaults=default" 83 | 84 | The output would be: 85 | 86 | .. code:: text 87 | 88 | (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in test-1/build 89 | (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in test-1/build 90 | (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.defaults, build in test-1/build 91 | 92 | .. _find-placeholders: 93 | 94 | **************************** 95 | ``find`` With Placeholders 96 | **************************** 97 | 98 | As you may notice in the earlier examples, ``idf-build-apps`` by default builds projects in-place, within the project directory, and generates the binaries under ``build`` directory (which is the default build directory for ESP-IDF projects). This makes it difficult to build all applications at the same time and keep the build artifacts separate in CI/CD pipelines. 99 | 100 | ``idf-build-apps`` supports placeholders to specify the build directory. The placeholders are replaced with the actual values during the call. The supported placeholders are: 101 | 102 | - ``@t``: Would be replaced by the target chip type. 103 | - ``@w``: Would be replaced by the wildcard if exists, otherwise would be replaced by the config name. 104 | - ``@n``: Would be replaced by the project name. 105 | - ``@f``: Would be replaced by the escaped project path (replaced "/" to "_"). 106 | - ``@v``: Would be replaced by the ESP-IDF version like ``5_3_0``. 107 | 108 | For example, 109 | 110 | .. code:: shell 111 | 112 | idf-build-apps find -p . --recursive --target esp32 --config "sdkconfig.ci.*=" --build-dir build_@t_@w 113 | 114 | The output would be: 115 | 116 | .. code:: text 117 | 118 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in ./test-1/build_esp32_bar 119 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in ./test-1/build_esp32_foo 120 | (cmake) App ./test-2, target esp32, sdkconfig (default), build in ./test-2/build_esp32 121 | 122 | Another example to build these apps in a temporary directory: 123 | 124 | .. code:: shell 125 | 126 | idf-build-apps find -p . --recursive --target esp32 --config "sdkconfig.ci.*=" --build-dir /tmp/build_@n_@t_@w 127 | 128 | The output would be: 129 | 130 | .. code:: text 131 | 132 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.bar, build in /tmp/build_test-1_esp32_bar 133 | (cmake) App ./test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in /tmp/build_test-1_esp32_foo 134 | (cmake) App ./test-2, target esp32, sdkconfig (default), build in /tmp/build_test-2_esp32 135 | 136 | ****************** 137 | Output to a File 138 | ****************** 139 | 140 | For `find` command, we support both "raw" format, and "json" format. The default format is "raw". 141 | 142 | In "raw" format, each line of the output represents an app, which is a JSON string that could be deserialized to an `App` object. 143 | 144 | .. code:: python 145 | 146 | from idf_build_apps import AppDeserializer 147 | 148 | with open("output.txt", "r") as f: 149 | for line in f: 150 | app = AppDeserializer.from_json(line) 151 | 152 | In "json" format, the output is a JSON array of `App` objects. 153 | 154 | To save the output to a file in "json" format, you can either pass the filename endswith "json", or use the ``--output-format json`` option. 155 | 156 | .. code:: shell 157 | 158 | idf-build-apps find --recursive --output output.json 159 | idf-build-apps find --recursive --output file --output-format json 160 | -------------------------------------------------------------------------------- /docs/en/guides/1.x_to_2.x.md: -------------------------------------------------------------------------------- 1 | # Migration From 1.x to 2.x 2 | 3 | There are a few breaking changes in 2.x. This document will help you migrate your code from 1.x to 2.x. 4 | 5 | ## Python Version Support 6 | 7 | idf-build-apps 1.x supports Python 2.7 and Python 3.4 or newer. idf-build-apps 2.x only supports Python 3.7 or newer. 8 | 9 | ## Logging Related Changes 10 | 11 | In 2.x, we're following the standard Python logging convention. 12 | 13 | Before: 14 | 15 | ```python 16 | from idf_build_apps import LOGGER 17 | ``` 18 | 19 | After: 20 | 21 | ```python 22 | import logging 23 | idf_build_apps_logger = logging.getLogger('idf_build_apps') 24 | ``` 25 | 26 | ## Normal Arguments to Keyword-only Arguments 27 | 28 | In 2.x, we move some arguments from normal arguments to keyword-only arguments. This is to make the API more consistent and easier to use. 29 | 30 | To understand the difference between these terms better, here's a quick summary: 31 | - "positonal-only argument" means the argument is a positional-only argument. (python 3.8+ only) 32 | - "keyword-only argument" means the argument is a keyword-only argument. 33 | - "normal argument" means the argument is not a positional-only argument, nor a keyword-only argument. 34 | 35 | For example, in the following function: 36 | 37 | ```python 38 | def foo(a, /, b, c, *, d=1, e=2, f=3): 39 | pass 40 | ``` 41 | 42 | - "a" is a positional-only argument. 43 | - "b" and "c" are normal arguments. 44 | - "d", "e", and "f" are keyword-only arguments. 45 | 46 | The following calls are valid: 47 | 48 | ```python 49 | foo(1, 2, 3, d=4, e=5, f=6) 50 | foo(1, 2, c=3, d=4, e=5, f=6) 51 | foo(1, b=2, c=3, d=4, e=5, f=6) 52 | ``` 53 | 54 | The following calls are invalid: 55 | 56 | ```python 57 | foo(1, 2, 3, 4, 5, 6) 58 | foo(1, b=2, 3, d=4, e=5, f=6) 59 | foo(a=1, b=2, c=3, d=4, e=5, f=6) 60 | ``` 61 | 62 | ### `App.__init__()` 63 | 64 | The `__init__` function of `App` class, and all its sub-classes, like `CMakeApp`, and `MakeApp`, now takes only `app_dir`, and `target` as normal arguments. All the rest of the arguments are keyword-only arguments. 65 | 66 | Before: 67 | 68 | ```python 69 | app = App('foo', 'esp32', 'sdkconfig.ci', 'default') 70 | ``` 71 | 72 | After: 73 | 74 | ```python 75 | app = App('foo', 'esp32', sdkconfig_path='sdkconfig.ci', config_name='default') 76 | ``` 77 | 78 | or all in keyword-only arguments: 79 | 80 | ```python 81 | app = App(app_dir='foo', target='esp32', sdkconfig_path='sdkconfig.ci', config_name='default') 82 | ``` 83 | 84 | ### `App.build()` 85 | 86 | The `build` function of `App` class, and all its sub-classes, like `CMakeApp`, and `MakeApp`, now takes all arguments as keyword-only arguments. 87 | 88 | ### `find_apps()` 89 | 90 | The `find_apps` function now takes only `paths` and `target` as normal arguments. All the rest of the arguments are keyword-only arguments. 91 | 92 | ### `build_apps()` 93 | 94 | The `build_apps` function now takes only `apps` as normal argument. All the rest of the arguments are keyword-only arguments. 95 | 96 | ## Function Signature Changes 97 | 98 | In 2.x, we change the signature of some functions to make them more intuitive and self-explanatory. 99 | 100 | ### `find_apps()` 101 | 102 | - `build_log_path` is renamed to `build_log_filename`. The file will be generated under `build_dir` if specified. 103 | - `size_json_path` is renamed to `size_json_filename`. The file will be generated under `build_dir` if specified. 104 | 105 | ## CLI Changes 106 | 107 | In 2.x, we change the separator for some options to better differentiate them from `None` and empty list. 108 | 109 | - `--modified-components` 110 | - `--modified-files` 111 | - `--ignore-app-dependencies-filepatterns` 112 | 113 | Before: 114 | 115 | ```shell 116 | idf-build-apps build -p foo -t esp32 --modified-components foo bar --modified-files foo bar --ignore-app-dependencies-filepatterns foo bar 117 | ``` 118 | 119 | After: 120 | 121 | ```shell 122 | idf-build-apps build -p foo -t esp32 --modified-components 'foo;bar' --modified-files 'foo;bar' --ignore-app-dependencies-filepatterns 'foo;bar' 123 | ``` 124 | 125 | passing `''` to specify it as `None` 126 | 127 | ```shell 128 | idf-build-apps build -p foo -t esp32 --modified-components '' 129 | ``` 130 | 131 | or passing `';'` to specify it as an empty list 132 | 133 | ```shell 134 | idf-build-apps build -p foo -t esp32 --modified-components ';' 135 | ``` 136 | -------------------------------------------------------------------------------- /docs/en/guides/custom_app.md: -------------------------------------------------------------------------------- 1 | # Custom App Classes 2 | 3 | `idf-build-apps` allows you to create custom app classes by subclassing the base `App` class. This is useful when you need to implement custom build logic or handle special project types. 4 | 5 | ## Creating a Custom App Class 6 | 7 | Here's an example of creating a custom app class: 8 | 9 | ```python 10 | from idf_build_apps import App 11 | from idf_build_apps.constants import BuildStatus 12 | import os 13 | from typing import Literal # Python 3.8+ only. from typing_extensions import Literal for earlier versions 14 | 15 | class CustomApp(App): 16 | build_system: Literal['custom'] = 'custom' # Must be unique to identify your custom app type 17 | 18 | def build(self, *args, **kwargs): 19 | # Implement your custom build logic here 20 | os.makedirs(self.build_path, exist_ok=True) 21 | with open(os.path.join(self.build_path, 'dummy.txt'), 'w') as f: 22 | f.write('Custom build successful') 23 | self.build_status = BuildStatus.SUCCESS 24 | print('Custom build successful') 25 | 26 | @classmethod 27 | def is_app(cls, path: str) -> bool: 28 | # Implement logic to determine if a path contains your custom app type 29 | return True 30 | ``` 31 | 32 | ## Using Custom App Classes 33 | 34 | You can use custom app classes in two ways: 35 | 36 | ### Via CLI 37 | 38 | ```shell 39 | idf-build-apps build -p /path/to/app --target esp32 --build-system custom:CustomApp 40 | ``` 41 | 42 | Where `custom:CustomApp` is in the format `module:class`. The module must be in your Python path. 43 | 44 | ### Via Python API 45 | 46 | ```python 47 | from idf_build_apps import find_apps 48 | 49 | apps = find_apps( 50 | paths=['/path/to/app'], 51 | target='esp32', 52 | build_system=CustomApp, 53 | ) 54 | 55 | for app in apps: 56 | app.build() 57 | ``` 58 | 59 | ## Important Notes 60 | 61 | - Your custom app class must subclass `App` 62 | - The `build_system` attribute must be unique to identify your app type 63 | - You must implement the `is_app()` class method to identify your app type 64 | - For JSON serialization support, you need to pass your custom class to `json_to_app()` when deserializing 65 | 66 | ## Example: JSON Serialization 67 | 68 | ```python 69 | from idf_build_apps import json_to_app 70 | 71 | # Serialize 72 | json_str = custom_app.to_json() 73 | 74 | # Deserialize 75 | deserialized_app = json_to_app(json_str, extra_classes=[CustomApp]) 76 | ``` 77 | 78 | ## Available Methods and Properties 79 | 80 | Please refer to the [API reference of the class `App`](https://docs.espressif.com/projects/idf-build-apps/en/latest/references/api/idf_build_apps.html#idf_build_apps.app.App) 81 | -------------------------------------------------------------------------------- /docs/en/index.rst: -------------------------------------------------------------------------------- 1 | ######################################## 2 | idf-build-apps |version| Documentation 3 | ######################################## 4 | 5 | This documentation is for idf-build-apps. idf-build-apps is a tool that allows developers to easily and reliably build applications for the ESP-IDF framework. 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | :caption: Explanations 10 | 11 | explanations/config_rules 12 | explanations/find 13 | explanations/build 14 | explanations/dependency_driven_build 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | :caption: Guides 19 | :glob: 20 | 21 | guides/* 22 | 23 | .. toctree:: 24 | :maxdepth: 1 25 | :caption: References 26 | 27 | references/manifest 28 | references/config_file 29 | references/cli 30 | references/api/modules.rst 31 | 32 | .. toctree:: 33 | :maxdepth: 1 34 | :caption: Others 35 | :glob: 36 | 37 | others/* 38 | -------------------------------------------------------------------------------- /docs/en/others/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ```{include} ../../../CHANGELOG.md 2 | ``` 3 | -------------------------------------------------------------------------------- /docs/en/others/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ```{include} ../../../CONTRIBUTING.md 2 | ``` 3 | -------------------------------------------------------------------------------- /docs/en/references/cli.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | CLI Reference 3 | ############### 4 | 5 | .. note:: 6 | 7 | All CLI options could be defined in the config file. For more information, please refer to the :doc:`Config File Reference <./config_file>`. 8 | 9 | .. argparse:: 10 | :ref: idf_build_apps.main.get_parser 11 | :prog: idf-build-apps 12 | -------------------------------------------------------------------------------- /docs/en/references/config_file.rst: -------------------------------------------------------------------------------- 1 | ############################################# 2 | Configuration File ``.idf_build_apps.toml`` 3 | ############################################# 4 | 5 | There are many CLI options available for ``idf-build-apps``. While these options provide usage flexibility, they also make the CLI command too long and difficult to read. However, a configuration file allows defining all these options in a more readable and maintainable way. 6 | 7 | *********************** 8 | Config File Discovery 9 | *********************** 10 | 11 | ``idf-build-apps`` supports a few ways to specify the configuration file (in order of precedence): 12 | 13 | - specify via CLI argument ``--config-file `` 14 | - ``.idf_build_apps.toml`` in the current directory 15 | - ``.idf_build_apps.toml`` in the parent directories, until it reaches the root of the file system 16 | - ``pyproject.toml`` with ``[tool.idf-build-apps]`` section 17 | - ``pyproject.toml`` in the parent directories, until it reaches the root of the file system 18 | 19 | ******* 20 | Usage 21 | ******* 22 | 23 | We recommend using the ``.idf_build_apps.toml`` file for non-Python projects and the ``pyproject.toml`` file for Python projects. When using the ``pyproject.toml`` file, define the configuration options in the ``[tool.idf-build-apps]`` section. 24 | 25 | Here's a simple example of a configuration file: 26 | 27 | .. tabs:: 28 | 29 | .. group-tab:: 30 | 31 | ``.idf_build_apps.toml`` 32 | 33 | .. code:: toml 34 | 35 | paths = [ 36 | "components", 37 | "examples", 38 | ] 39 | target = "esp32" 40 | recursive = true 41 | 42 | # config rules 43 | config = [ 44 | "sdkconfig.*=", 45 | "=default", 46 | ] 47 | 48 | .. group-tab:: 49 | 50 | ``pyproject.toml`` 51 | 52 | .. code:: toml 53 | 54 | [tool.idf-build-apps] 55 | paths = [ 56 | "components", 57 | "examples", 58 | ] 59 | target = "esp32" 60 | recursive = true 61 | 62 | # config rules 63 | config = [ 64 | "sdkconfig.*=", 65 | "=default", 66 | ] 67 | 68 | Running ``idf-build-apps build`` with the above configuration is equivalent to the following CLI command: 69 | 70 | .. code:: shell 71 | 72 | idf-build-app build \ 73 | --paths components examples \ 74 | --target esp32 \ 75 | --recursive \ 76 | --config-rules "sdkconfig.*=" "=default" \ 77 | --build-dir "build_@t_@w" 78 | 79 | `TOML `__ supports native data types. In order to get the config name and type of the corresponding CLI option, you may refer to the help messages by using ``idf-build-apps find -h`` or ``idf-build-apps build -h``. 80 | 81 | For instance, the ``--paths`` CLI option help message shows: 82 | 83 | .. code:: text 84 | 85 | -p PATHS [PATHS ...], --paths PATHS [PATHS ...] 86 | One or more paths to look for apps. 87 | - default: None 88 | - config name: paths 89 | - config type: list[str] 90 | 91 | This indicates that in the configuration file, you should specify it with the name ``paths``, and the type should be a “list of strings”. 92 | 93 | .. code:: toml 94 | 95 | paths = [ 96 | "foo", 97 | "bar", 98 | ] 99 | 100 | ****************************** 101 | Expand Environment Variables 102 | ****************************** 103 | 104 | All configuration options support environment variables. You can use environment variables in the configuration file by using the syntax ``${VAR_NAME}`` or ``$VAR_NAME``. Undeclared environment variables will be replaced with an empty string. For example: 105 | 106 | .. code:: toml 107 | 108 | collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}" 109 | 110 | when the environment variable ``CI_JOB_NAME_SLUG`` is set to ``my_job``, the ``collect_app_info_filename`` will be expanded to ``app_info_my_job``. When the environment variable is not set, the value will be ``app_info_``. 111 | 112 | ************************* 113 | CLI Argument Precedence 114 | ************************* 115 | 116 | CLI arguments take precedence over the configuration file. This helps to override the configuration file settings when required. 117 | 118 | For example, if the configuration file has the following content: 119 | 120 | .. tabs:: 121 | 122 | .. group-tab:: 123 | 124 | ``.idf_build_apps.toml`` 125 | 126 | .. code:: toml 127 | 128 | target = "esp32" 129 | config_rules = [ 130 | "sdkconfig.*=", 131 | "=default", 132 | ] 133 | 134 | .. group-tab:: 135 | 136 | ``pyproject.toml`` 137 | 138 | .. code:: toml 139 | 140 | [tool.idf-build-apps] 141 | target = "esp32" 142 | config_rules = [ 143 | "sdkconfig.*=", 144 | "=default", 145 | ] 146 | 147 | Override String Configuration 148 | ============================= 149 | 150 | To override the ``str`` type configuration, (e.g., ``target``), you can pass the CLI argument directly: 151 | 152 | .. code:: shell 153 | 154 | idf-build-apps build --target esp32s2 155 | 156 | Override List Configuration 157 | =========================== 158 | 159 | To override the ``list[str]`` type configuration, (e.g., ``config_rules``), you can override it by passing the CLI argument. For example: 160 | 161 | .. code:: shell 162 | 163 | idf-build-apps build --config-rules "foo=bar" 164 | 165 | Or you can unset the configuration by passing an empty string: 166 | 167 | .. code:: shell 168 | 169 | idf-build-apps build --config-rules "" 170 | 171 | Override Boolean Configuration 172 | ============================== 173 | 174 | Not supported yet. 175 | -------------------------------------------------------------------------------- /docs/en/references/manifest.rst: -------------------------------------------------------------------------------- 1 | ######################################### 2 | Manifest File ``.build-test-rules.yml`` 3 | ######################################### 4 | 5 | A ``.build-test-rules.yml`` file is the manifest file to control whether the app will be built or tested under the rules. 6 | 7 | One typical manifest file look like this: 8 | 9 | .. code:: yaml 10 | 11 | [folder]: 12 | enable: 13 | - if: [if clause] 14 | temporary: true # optional, default to false. `reason` is required if `temporary` is true 15 | reason: [your reason] # optional 16 | - ... 17 | disable: 18 | - if: [if clause] 19 | - ... 20 | disable_test: 21 | - if: [if clause] 22 | - ... 23 | 24 | ******* 25 | Terms 26 | ******* 27 | 28 | Supported Targets 29 | ================= 30 | 31 | This refers to the targets that are fully supported by the ESP-IDF project. You may check the supported targets by running ``idf.py --list-targets``. 32 | 33 | ``idf-build-apps`` will get this information dynamically from your ``$IDF_PATH``. For ESP-IDF release 5.3, the supported targets are: 34 | 35 | - esp32 36 | - esp32s2 37 | - esp32c3 38 | - esp32s3 39 | - esp32c2 40 | - esp32c6 41 | - esp32h2 42 | - esp32p4 43 | 44 | Preview Targets 45 | =============== 46 | 47 | This refers to the targets that are still in preview status. You may check the preview targets by running ``idf.py --list-targets --preview``. 48 | 49 | ``idf-build-apps`` will get this information dynamically from your ``$IDF_PATH``. For ESP-IDF release 5.3, the preview targets are: 50 | 51 | - linux 52 | - esp32c5 53 | - esp32c61 54 | 55 | **************** 56 | ``if`` Clauses 57 | **************** 58 | 59 | Operands 60 | ======== 61 | 62 | - Capitalized Words 63 | 64 | - Variables defined in ``IDF_PATH/components/soc/[TARGET]/include/soc/*_caps.h`` or in ``IDF_PATH/components/esp_rom/[TARGET]/*_caps.h``. e.g., ``SOC_WIFI_SUPPORTED``, ``ESP_ROM_HAS_SPI_FLASH`` 65 | - ``IDF_TARGET`` 66 | - ``IDF_VERSION`` (IDF_VERSION_MAJOR.IDF_VERSION_MINOR.IDF_VERSION_PATCH. e.g., 5.2.0. Will convert to Version object to do a version comparison instead of a string comparison) 67 | - ``IDF_VERSION_MAJOR`` 68 | - ``IDF_VERSION_MINOR`` 69 | - ``IDF_VERSION_PATCH`` 70 | - ``INCLUDE_DEFAULT`` (The default value of supported targets is 1, and the default value of preview targets is 0) 71 | - ``CONFIG_NAME`` (config name defined in :doc:`../explanations/config_rules`) 72 | - environment variables, default to ``0`` if not set 73 | 74 | - String, must be double-quoted. e.g., ``"esp32"``, ``"12345"`` 75 | 76 | - Integer, support decimal and hex. e.g., ``1``, ``0xAB`` 77 | 78 | - List of strings or integers, or both types at the same time. e.g., ``["esp32", 1]`` 79 | 80 | Operators 81 | ========= 82 | 83 | - ``==``, ``!=``, ``>``, ``>=``, ``<``, ``<=`` 84 | - ``and``, ``or`` 85 | - ``in``, ``not in`` with list 86 | - parentheses 87 | 88 | Limitations 89 | =========== 90 | 91 | All operators are binary operators. For more than two operands, you may use the nested parentheses trick. For example: 92 | 93 | - ``A == 1 or (B == 2 and C in [1,2,3])`` 94 | - ``(A == 1 and B == 2) or (C not in ["3", "4", 5])`` 95 | 96 | .. warning:: 97 | 98 | Chained ``and`` and ``or`` operators are not supported. The operands start from the third one will be ignored. 99 | 100 | For example, ``A == 1 and B == 2 and C == 3`` will be interpreted as ``A == 1 and B == 2``. 101 | 102 | ********************** 103 | Enable/Disable Rules 104 | ********************** 105 | 106 | By default, we enable build and test for all supported targets. In other words, all preview targets are disabled. 107 | 108 | To simplify the manifest file, if an app needs to be build and tested on all supported targets, it does not need to be added in a manifest file. The manifest files are files that set the violation rules for apps. 109 | 110 | Three rules (disable rules are calculated after the enable rules): 111 | 112 | - ``enable``: run CI build/test jobs for targets that match any of the specified conditions only 113 | - ``disable``: will not run CI build/test jobs for targets that match any of the specified conditions 114 | - ``disable_test``: will not run CI test jobs for targets that match any of the specified conditions 115 | 116 | Each key is a folder. The rule will recursively apply to all apps inside. 117 | 118 | Overrides Rules 119 | =============== 120 | 121 | If one sub folder is in a special case, you can overwrite the rules for this folder by adding another entry for this folder itself. Each folder’s rules are standalone, and will not inherit its parent’s rules. (YAML inheritance is too complicated for reading) 122 | 123 | For example, in the following code block, only ``disable`` rule exists in ``examples/foo/bar``. It’s unaware of its parent’s ``enable`` rule. 124 | 125 | .. code:: yaml 126 | 127 | examples/foo: 128 | enable: 129 | - if: IDF_TARGET == "esp32" 130 | 131 | examples/foo/bar: 132 | disable: 133 | - if: IDF_TARGET == "esp32s2" 134 | 135 | ******************* 136 | Practical Example 137 | ******************* 138 | 139 | Here’s a practical example: 140 | 141 | .. code:: yaml 142 | 143 | examples/foo: 144 | enable: 145 | - if IDF_TARGET in ["esp32", 1, 2, 3] 146 | - if IDF_TARGET not in ["4", "5", 6] 147 | # should be run under all targets! 148 | 149 | examples/bluetooth: 150 | disable: # disable both build and tests jobs 151 | - if: SOC_BT_SUPPORTED != 1 152 | # reason is optional if there's no `temporary: true` 153 | disable_test: 154 | - if: IDF_TARGET == "esp32" 155 | temporary: true 156 | reason: lack of ci runners # required when `temporary: true` 157 | 158 | examples/bluetooth/test_foo: 159 | # each folder's settings are standalone 160 | disable: 161 | - if: IDF_TARGET == "esp32s2" 162 | temporary: true 163 | reason: no idea 164 | # unlike examples/bluetooth, the apps under this folder would not be build nor test for "no idea" under target esp32s2 165 | 166 | examples/get-started/hello_world: 167 | enable: 168 | - if: IDF_TARGET == "linux" 169 | reason: this one only supports linux! 170 | 171 | examples/get-started/blink: 172 | enable: 173 | - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" 174 | reason: This one supports all supported targets and linux 175 | 176 | ********************** 177 | Enhanced YAML Syntax 178 | ********************** 179 | 180 | Switch-Like Clauses 181 | =================== 182 | 183 | The Switch-Like clauses are supported by two keywords in the YAML file: ``depends_components`` and ``depends_filepatterns``. 184 | 185 | Operands 186 | -------- 187 | 188 | Switch cases have two main components: the ``if`` clause and the ``default`` clause. Just like a switch statement in c language, The first matched ``if`` clause will be applied. If no ``if`` clause matched, the ``default`` clause will be applied. Here’s an example: 189 | 190 | .. code:: yaml 191 | 192 | test1: 193 | depends_components: 194 | - if: IDF_VERSION == "{IDF_VERSION}" 195 | content: [ "component_1" ] 196 | - if: CONFIG_NAME == "AWESOME_CONFIG" 197 | content: [ "component_2" ] 198 | - default: [ "component_3", "component_4" ] 199 | 200 | ``default`` clause is optional. If you don’t specify any ``default`` clause, it will return an empty array. 201 | 202 | Limitations 203 | ----------- 204 | 205 | You cannot combine a list and a switch in one node. 206 | 207 | Reuse Lists 208 | =========== 209 | 210 | To reuse the items defined in a list, you can use the ``+`` and ``-`` postfixes respectively. The order of calculation is always ``+`` first, followed by ``-``. 211 | 212 | Array Elements as Strings 213 | ------------------------- 214 | 215 | The following YAML code demonstrates how to reuse the elements from a list of strings: 216 | 217 | .. code:: yaml 218 | 219 | .base_depends_components: &base-depends-components 220 | depends_components: 221 | - esp_hw_support 222 | - esp_rom 223 | - esp_wifi 224 | 225 | examples/wifi/coexist: 226 | <<: *base-depends-components 227 | depends_components+: 228 | - esp_coex 229 | depends_components-: 230 | - esp_rom 231 | 232 | After interpretation, the resulting YAML will be: 233 | 234 | .. code:: yaml 235 | 236 | examples/wifi/coexist: 237 | depends_components: 238 | - esp_hw_support 239 | - esp_wifi 240 | - esp_coex 241 | 242 | This means that the ``esp_rom`` element is removed, and the ``esp_coex`` element is added to the ``depends_components`` list. 243 | 244 | Array Elements as Dictionaries 245 | ------------------------------ 246 | 247 | In addition to reuse elements from a list of strings, you can also perform these operations on a list of dictionaries. The matching is done based on the ``if`` key. Here’s an example: 248 | 249 | .. code:: yaml 250 | 251 | .base: &base 252 | enable: 253 | - if: IDF_VERSION == "5.2.0" 254 | - if: IDF_VERSION == "5.3.0" 255 | 256 | foo: 257 | <<: *base 258 | enable+: 259 | # this if statement dictionary will override the one defined in `&base` 260 | - if: IDF_VERSION == "5.2.0" 261 | temp: true 262 | - if: IDF_VERSION == "5.4.0" 263 | reason: bar 264 | 265 | After interpretation, the resulting YAML will be: 266 | 267 | .. code:: yaml 268 | 269 | foo: 270 | enable: 271 | - if: IDF_VERSION == "5.3.0" 272 | - if: IDF_VERSION == "5.2.0" 273 | temp: true 274 | - if: IDF_VERSION == "5.4.0" 275 | reason: bar 276 | 277 | In this case, the ``enable`` list is extended with the new ``if`` statement and ``reason`` dictionary. 278 | 279 | It’s important to note that the ``if`` dictionary defined in the ``+`` postfix will override the earlier one when the ``if`` statement matches. 280 | 281 | This demonstrates how you can use the ``+`` and ``-`` postfixes to extend and remove elements from both string and dictionary lists in our YAML. 282 | -------------------------------------------------------------------------------- /idf_build_apps/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """ 5 | Tools for building ESP-IDF related apps. 6 | """ 7 | 8 | # ruff: noqa: E402 9 | # avoid circular imports 10 | 11 | __version__ = '2.11.2' 12 | 13 | from .session_args import ( 14 | SessionArgs, 15 | ) 16 | 17 | SESSION_ARGS = SessionArgs() 18 | 19 | from .app import ( 20 | App, 21 | AppDeserializer, 22 | CMakeApp, 23 | MakeApp, 24 | ) 25 | from .log import ( 26 | setup_logging, 27 | ) 28 | from .main import ( 29 | build_apps, 30 | find_apps, 31 | json_list_files_to_apps, 32 | json_to_app, 33 | ) 34 | 35 | __all__ = [ 36 | 'App', 37 | 'AppDeserializer', 38 | 'CMakeApp', 39 | 'MakeApp', 40 | 'build_apps', 41 | 'find_apps', 42 | 'json_list_files_to_apps', 43 | 'json_to_app', 44 | 'setup_logging', 45 | ] 46 | -------------------------------------------------------------------------------- /idf_build_apps/__main__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from .main import ( 5 | main, 6 | ) 7 | 8 | if __name__ == '__main__': 9 | main() 10 | -------------------------------------------------------------------------------- /idf_build_apps/autocompletions.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | from typing import Optional 6 | 7 | from .utils import AutocompleteActivationError 8 | 9 | 10 | def append_to_file(file_path: str, content: str) -> None: 11 | """Add commands to shell configuration file 12 | 13 | :param file_path: path to shell configurations file 14 | :param content: commands to add 15 | """ 16 | if os.path.exists(file_path): 17 | with open(file_path) as file: 18 | if content.strip() in file.read(): 19 | print(f'Autocompletion already set up in {file_path}') 20 | return 21 | with open(file_path, 'a') as file: 22 | file.write(f'\n# Begin added by idf-build-apps \n{content} \n# End added by idf-build-apps') 23 | print(f'Autocompletion added to {file_path}') 24 | 25 | 26 | def activate_completions(shell_type: Optional[str]) -> None: 27 | """Activates autocompletion for supported shells. 28 | 29 | :raises AutocompleteActivationError: if the $SHELL env variable is empty, or if the detected shell is unsupported. 30 | """ 31 | supported_shells = ['bash', 'zsh', 'fish'] 32 | 33 | if shell_type == 'auto': 34 | shell_type = os.path.basename(os.environ.get('SHELL', '')) 35 | 36 | if not shell_type: 37 | raise AutocompleteActivationError('$SHELL is empty. Please provide your shell type with the `--shell` option') 38 | 39 | if shell_type not in supported_shells: 40 | raise AutocompleteActivationError('Unsupported shell. Autocompletion is supported for bash, zsh and fish.') 41 | 42 | if shell_type == 'bash': 43 | completion_command = 'eval "$(register-python-argcomplete idf-build-apps)"' 44 | elif shell_type == 'zsh': 45 | completion_command = ( 46 | 'autoload -U bashcompinit && bashcompinit && eval "$(register-python-argcomplete idf-build-apps)"' 47 | ) 48 | elif shell_type == 'fish': 49 | completion_command = 'register-python-argcomplete --shell fish idf-build-apps | source' 50 | 51 | rc_file = {'bash': '~/.bashrc', 'zsh': '~/.zshrc', 'fish': '~/.config/fish/completions/idf-build-apps.fish'} 52 | 53 | shell_rc = os.path.expanduser(rc_file[shell_type]) 54 | append_to_file(shell_rc, completion_command) 55 | -------------------------------------------------------------------------------- /idf_build_apps/constants.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import enum 5 | import os 6 | 7 | import esp_bool_parser 8 | 9 | IDF_PATH = esp_bool_parser.IDF_PATH 10 | IDF_PY = os.path.join(IDF_PATH, 'tools', 'idf.py') 11 | IDF_SIZE_PY = os.path.join(IDF_PATH, 'tools', 'idf_size.py') 12 | 13 | PROJECT_DESCRIPTION_JSON = 'project_description.json' 14 | DEFAULT_SDKCONFIG = 'sdkconfig.defaults' 15 | 16 | SUPPORTED_TARGETS = esp_bool_parser.SUPPORTED_TARGETS 17 | PREVIEW_TARGETS = esp_bool_parser.PREVIEW_TARGETS 18 | ALL_TARGETS = esp_bool_parser.ALL_TARGETS 19 | IDF_VERSION_MAJOR = esp_bool_parser.IDF_VERSION_MAJOR 20 | IDF_VERSION_MINOR = esp_bool_parser.IDF_VERSION_MINOR 21 | IDF_VERSION_PATCH = esp_bool_parser.IDF_VERSION_PATCH 22 | IDF_VERSION = esp_bool_parser.IDF_VERSION 23 | 24 | 25 | class BuildStatus(str, enum.Enum): 26 | UNKNOWN = 'unknown' 27 | DISABLED = 'disabled' 28 | SKIPPED = 'skipped' 29 | SHOULD_BE_BUILT = 'should be built' 30 | FAILED = 'build failed' 31 | SUCCESS = 'build success' 32 | 33 | 34 | completion_instructions = """ 35 | With the `--activate` option, detect your shell type and add the appropriate commands to your shell's config file 36 | so that it runs on startup. You will likely have to restart. 37 | or re-login for the autocompletion to start working. 38 | 39 | You can also specify your shell using the `--shell` option. 40 | 41 | If you do not want automatic modification of your shell configuration file 42 | You can manually add the commands provided below to activate autocompletion. 43 | or run them in your current terminal session for one-time activation. 44 | 45 | Once again, you will likely have to restart 46 | or re-login for the autocompletion to start working. 47 | 48 | bash: 49 | eval "$(register-python-argcomplete idf-build-apps)" 50 | 51 | zsh: 52 | To activate completions in zsh, first make sure compinit is marked for 53 | autoload and run autoload: 54 | 55 | autoload -U compinit 56 | compinit 57 | 58 | Afterwards you can enable completions for idf-build-apps: 59 | 60 | eval "$(register-python-argcomplete idf-build-apps)" 61 | 62 | fish: 63 | # Not required to be in the config file, only run once 64 | register-python-argcomplete --shell fish idf-build-apps >~/.config/fish/completions/idf-build-apps.fish 65 | """ 66 | IDF_BUILD_APPS_TOML_FN = '.idf_build_apps.toml' 67 | -------------------------------------------------------------------------------- /idf_build_apps/finder.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import os 6 | import os.path 7 | import re 8 | import typing as t 9 | from pathlib import ( 10 | Path, 11 | ) 12 | 13 | from .app import ( 14 | App, 15 | CMakeApp, 16 | ) 17 | from .args import FindArguments 18 | from .constants import ( 19 | BuildStatus, 20 | ) 21 | from .manifest.manifest import DEFAULT_BUILD_TARGETS 22 | from .utils import ( 23 | config_rules_from_str, 24 | to_absolute_path, 25 | ) 26 | 27 | LOGGER = logging.getLogger(__name__) 28 | 29 | 30 | def _get_apps_from_path( 31 | path: str, 32 | target: str, 33 | *, 34 | app_cls: t.Type[App] = CMakeApp, 35 | args: FindArguments, 36 | ) -> t.List[App]: 37 | def _validate_app(_app: App) -> bool: 38 | if target not in _app.supported_targets: 39 | LOGGER.debug('=> Ignored. %s only supports targets: %s', _app, ', '.join(_app.supported_targets)) 40 | _app.build_status = BuildStatus.DISABLED 41 | return args.include_disabled_apps 42 | 43 | if target == 'all' and _app.target not in DEFAULT_BUILD_TARGETS.get(): 44 | LOGGER.debug( 45 | '=> Ignored. %s is not in the default build targets: %s', _app.target, DEFAULT_BUILD_TARGETS.get() 46 | ) 47 | _app.build_status = BuildStatus.DISABLED 48 | return args.include_disabled_apps 49 | elif _app.target != target: 50 | LOGGER.debug('=> Ignored. %s is not for target %s', _app, target) 51 | _app.build_status = BuildStatus.DISABLED 52 | return args.include_disabled_apps 53 | 54 | _app.check_should_build( 55 | manifest_rootpath=args.manifest_rootpath, 56 | modified_manifest_rules_folders=args.modified_manifest_rules_folders, 57 | modified_components=args.modified_components, 58 | modified_files=args.modified_files, 59 | check_app_dependencies=args.dependency_driven_build_enabled, 60 | ) 61 | 62 | # for unknown ones, we keep them to the build stage to judge 63 | if _app.build_status == BuildStatus.SKIPPED: 64 | LOGGER.debug('=> Skipped. Reason: %s', _app.build_comment or 'Unknown') 65 | return args.include_skipped_apps 66 | 67 | return True 68 | 69 | if not app_cls.is_app(path): 70 | LOGGER.debug('Skipping. %s is not an app', path) 71 | return [] 72 | 73 | config_rules = config_rules_from_str(args.config_rules) 74 | 75 | apps = [] 76 | default_config_name = '' 77 | sdkconfig_paths_matched = False 78 | for rule in config_rules: 79 | if not rule.file_name: 80 | default_config_name = rule.config_name 81 | continue 82 | 83 | sdkconfig_paths = sorted([str(p.resolve()) for p in Path(path).glob(rule.file_name)]) 84 | 85 | if sdkconfig_paths: 86 | sdkconfig_paths_matched = True # skip the next block for no wildcard config rules 87 | 88 | for sdkconfig_path in sdkconfig_paths: 89 | if sdkconfig_path.endswith(f'.{target}'): 90 | LOGGER.debug('=> Skipping sdkconfig %s which is target-specific', sdkconfig_path) 91 | continue 92 | 93 | # Figure out the config name 94 | config_name = rule.config_name or '' 95 | if '*' in rule.file_name: 96 | # convert glob pattern into a regex 97 | regex_str = r'.*' + rule.file_name.replace('.', r'\.').replace('*', r'(.*)') 98 | groups = re.match(regex_str, sdkconfig_path) 99 | assert groups 100 | config_name = groups.group(1) 101 | 102 | app = app_cls( 103 | path, 104 | target, 105 | sdkconfig_path=sdkconfig_path, 106 | config_name=config_name, 107 | work_dir=args.work_dir, 108 | build_dir=args.build_dir, 109 | build_log_filename=args.build_log_filename, 110 | size_json_filename=args.size_json_filename, 111 | check_warnings=args.check_warnings, 112 | sdkconfig_defaults_str=args.sdkconfig_defaults, 113 | ) 114 | if _validate_app(app): 115 | LOGGER.debug('Found app: %s', app) 116 | apps.append(app) 117 | 118 | LOGGER.debug('') # add one empty line for separating different finds 119 | 120 | # no config rules matched, use default app 121 | if not sdkconfig_paths_matched: 122 | app = app_cls( 123 | path, 124 | target, 125 | sdkconfig_path=None, 126 | config_name=default_config_name, 127 | work_dir=args.work_dir, 128 | build_dir=args.build_dir, 129 | build_log_filename=args.build_log_filename, 130 | size_json_filename=args.size_json_filename, 131 | check_warnings=args.check_warnings, 132 | sdkconfig_defaults_str=args.sdkconfig_defaults, 133 | ) 134 | 135 | if _validate_app(app): 136 | LOGGER.debug('Found app: %s', app) 137 | apps.append(app) 138 | 139 | LOGGER.debug('') # add one empty line for separating different finds 140 | 141 | return sorted(apps) 142 | 143 | 144 | def _find_apps( 145 | path: str, 146 | target: str, 147 | *, 148 | app_cls: t.Type[App] = CMakeApp, 149 | args: FindArguments, 150 | ) -> t.List[App]: 151 | LOGGER.debug( 152 | 'Looking for %s apps in %s%s with target %s', 153 | app_cls.__name__, 154 | path, 155 | ' recursively' if args.recursive else '', 156 | target, 157 | ) 158 | 159 | if not args.recursive: 160 | if args.exclude: 161 | LOGGER.debug('--exclude option is ignored when used without --recursive') 162 | 163 | return _get_apps_from_path(path, target, app_cls=app_cls, args=args) 164 | 165 | # The remaining part is for recursive == True 166 | apps = [] 167 | # handle the exclude list, since the config file might use linux style, but run in windows 168 | exclude_paths_list = [to_absolute_path(p) for p in args.exclude or []] 169 | for root, dirs, _ in os.walk(path): 170 | LOGGER.debug('Entering %s', root) 171 | root_path = to_absolute_path(root) 172 | if root_path in exclude_paths_list: 173 | LOGGER.debug('=> Skipping %s (excluded)', root) 174 | del dirs[:] 175 | continue 176 | 177 | if os.path.basename(root_path) == 'managed_components': # idf-component-manager 178 | LOGGER.debug('=> Skipping %s (managed components)', root_path) 179 | del dirs[:] 180 | continue 181 | 182 | _found_apps = _get_apps_from_path(root, target, app_cls=app_cls, args=args) 183 | if _found_apps: # root has at least one app 184 | LOGGER.debug('=> Stop iteration sub dirs of %s since it has apps', root) 185 | del dirs[:] 186 | apps.extend(_found_apps) 187 | continue 188 | 189 | return apps 190 | -------------------------------------------------------------------------------- /idf_build_apps/junit/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from .report import ( 5 | TestCase, 6 | TestReport, 7 | TestSuite, 8 | ) 9 | 10 | __all__ = ['TestCase', 'TestReport', 'TestSuite'] 11 | -------------------------------------------------------------------------------- /idf_build_apps/junit/report.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """ 5 | The test report should look like something like this: 6 | 7 | .. code-block:: xml 8 | 9 | 10 | 19 | 23 | 24 | 25 | 29 | 30 | 31 | 32 | 33 | """ 34 | 35 | import json 36 | import logging 37 | import os.path 38 | import typing as t 39 | from datetime import ( 40 | datetime, 41 | timezone, 42 | ) 43 | from xml.etree import ( 44 | ElementTree, 45 | ) 46 | from xml.sax.saxutils import ( 47 | escape, 48 | ) 49 | 50 | from ..app import ( 51 | App, 52 | ) 53 | from ..constants import ( 54 | BuildStatus, 55 | ) 56 | from .utils import ( 57 | get_sys_info, 58 | ) 59 | 60 | LOGGER = logging.getLogger(__name__) 61 | 62 | 63 | class TestCase: 64 | def __init__( 65 | self, 66 | name: str, 67 | *, 68 | error_reason: t.Optional[str] = None, 69 | failure_reason: t.Optional[str] = None, 70 | skipped_reason: t.Optional[str] = None, 71 | properties: t.Optional[t.Dict[str, str]] = None, 72 | duration_sec: float = 0, 73 | timestamp: t.Optional[datetime] = None, 74 | ) -> None: 75 | self.name = name 76 | 77 | self.failure_reason = failure_reason 78 | self.skipped_reason = skipped_reason 79 | self.error_reason = error_reason 80 | # only have one reason among these three 81 | if sum([self.failure_reason is not None, self.skipped_reason is not None, self.error_reason is not None]) > 1: 82 | raise ValueError('Only one of failure_reason, skipped_reason, error_reason can be set') 83 | 84 | self.duration_sec = duration_sec 85 | self.timestamp = timestamp or datetime.now(timezone.utc) 86 | 87 | self.properties = properties or {} 88 | 89 | @classmethod 90 | def from_app(cls, app: App) -> 'TestCase': 91 | if app.build_status in (BuildStatus.UNKNOWN, BuildStatus.SHOULD_BE_BUILT): 92 | raise ValueError( 93 | f'Cannot create build report for apps with build status {app.build_status}. ' 94 | f'Please finish the build process first.' 95 | ) 96 | 97 | kwargs: t.Dict[str, t.Any] = { 98 | 'name': app.build_path, 99 | 'duration_sec': app._build_duration, 100 | 'timestamp': app._build_timestamp, 101 | 'properties': {}, 102 | } 103 | if app.build_status == BuildStatus.FAILED: 104 | kwargs['failure_reason'] = app.build_comment 105 | elif app.build_status in (BuildStatus.DISABLED, BuildStatus.SKIPPED): 106 | kwargs['skipped_reason'] = app.build_comment 107 | 108 | if app.size_json_path and os.path.isfile(app.size_json_path): 109 | with open(app.size_json_path) as f: 110 | for k, v in json.load(f).items(): 111 | kwargs['properties'][f'{k}'] = str(v) 112 | 113 | return cls(**kwargs) 114 | 115 | @property 116 | def is_failed(self) -> bool: 117 | return self.failure_reason is not None 118 | 119 | @property 120 | def is_skipped(self) -> bool: 121 | return self.skipped_reason is not None 122 | 123 | @property 124 | def is_error(self) -> bool: 125 | return self.error_reason is not None 126 | 127 | def to_xml_elem(self) -> ElementTree.Element: 128 | elem = ElementTree.Element( 129 | 'testcase', 130 | { 131 | 'name': self.name, 132 | 'time': str(self.duration_sec), 133 | 'timestamp': self.timestamp.isoformat(), 134 | }, 135 | ) 136 | if self.error_reason: 137 | ElementTree.SubElement(elem, 'error', {'message': escape(self.error_reason)}) 138 | elif self.failure_reason: 139 | ElementTree.SubElement(elem, 'failure', {'message': escape(self.failure_reason)}) 140 | elif self.skipped_reason: 141 | ElementTree.SubElement(elem, 'skipped', {'message': escape(self.skipped_reason)}) 142 | 143 | if self.properties: 144 | for k, v in self.properties.items(): 145 | elem.attrib[k] = escape(str(v)) 146 | 147 | return elem 148 | 149 | 150 | class TestSuite: 151 | def __init__(self, name: str) -> None: 152 | self.name = name 153 | 154 | self.test_cases: t.List[TestCase] = [] 155 | 156 | self.tests = 0 # passed, actually 157 | self.errors = 0 # setup error 158 | self.failures = 0 # runtime failures 159 | self.skipped = 0 160 | 161 | self.duration_sec: float = 0 162 | self.timestamp = datetime.now(timezone.utc) 163 | 164 | self.properties = get_sys_info() 165 | 166 | def add_test_case(self, test_case: TestCase) -> None: 167 | self.test_cases.append(test_case) 168 | 169 | if test_case.is_error: 170 | self.errors += 1 171 | elif test_case.is_failed: 172 | self.failures += 1 173 | elif test_case.is_skipped: 174 | self.skipped += 1 175 | else: 176 | self.tests += 1 177 | 178 | self.duration_sec += test_case.duration_sec 179 | 180 | def to_xml_elem(self) -> ElementTree.Element: 181 | elem = ElementTree.Element( 182 | 'testsuite', 183 | { 184 | 'name': self.name, 185 | 'tests': str(self.tests), 186 | 'errors': str(self.errors), 187 | 'failures': str(self.failures), 188 | 'skipped': str(self.skipped), 189 | 'time': str(self.duration_sec), 190 | 'timestamp': self.timestamp.isoformat(), 191 | **self.properties, 192 | }, 193 | ) 194 | 195 | for test_case in self.test_cases: 196 | elem.append(test_case.to_xml_elem()) 197 | 198 | return elem 199 | 200 | 201 | class TestReport: 202 | def __init__(self, test_suites: t.List[TestSuite], filepath: str) -> None: 203 | self.test_suites: t.List[TestSuite] = test_suites 204 | 205 | self.filepath = filepath 206 | 207 | def create_test_report(self) -> None: 208 | xml = ElementTree.Element('testsuites') 209 | 210 | for test_suite in self.test_suites: 211 | xml.append(test_suite.to_xml_elem()) 212 | 213 | ElementTree.ElementTree(xml).write(self.filepath, encoding='utf-8') 214 | LOGGER.info('Generated build junit report at: %s', self.filepath) 215 | -------------------------------------------------------------------------------- /idf_build_apps/junit/utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import platform 6 | import re 7 | import socket 8 | import sys 9 | import typing as t 10 | 11 | 12 | def get_size(b: float) -> str: 13 | for unit in ['', 'K', 'M', 'G', 'T', 'P']: 14 | if b < 1024: 15 | return f'{b:.2f}{unit}B' 16 | b /= 1024 17 | 18 | return f'{b:.2f}EB' 19 | 20 | 21 | def get_processor_name(): 22 | if platform.processor(): 23 | return platform.processor() 24 | 25 | # read from /proc/cpuinfo 26 | if os.path.isfile('/proc/cpuinfo'): 27 | try: 28 | with open('/proc/cpuinfo') as f: 29 | for line in f: 30 | if 'model name' in line: 31 | return re.sub('.*model name.*:', '', line, count=1).strip() 32 | except Exception: 33 | pass 34 | 35 | return '' 36 | 37 | 38 | def get_sys_info() -> t.Dict[str, str]: 39 | info = { 40 | 'platform': platform.system(), 41 | 'platform-release': platform.release(), 42 | 'architecture': platform.machine(), 43 | 'hostname': socket.gethostname(), 44 | 'processor': get_processor_name(), 45 | 'cpu_count': str(os.cpu_count()) if os.cpu_count() else 'Unknown', 46 | } 47 | 48 | if sys.platform != 'win32': 49 | info['ram'] = get_size(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')) 50 | 51 | return info 52 | -------------------------------------------------------------------------------- /idf_build_apps/log.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import typing as t 6 | from datetime import datetime 7 | 8 | from rich import get_console 9 | from rich._log_render import LogRender 10 | from rich.console import Console, ConsoleRenderable 11 | from rich.containers import Renderables 12 | from rich.logging import RichHandler 13 | from rich.text import Text, TextType 14 | 15 | 16 | class _OneLineLogRender(LogRender): 17 | def __call__( # type: ignore # the original method returns Table instead of Text 18 | self, 19 | console: Console, 20 | renderables: t.Iterable[ConsoleRenderable], 21 | log_time: t.Optional[datetime] = None, 22 | time_format: t.Optional[t.Union[str, t.Callable[[datetime], Text]]] = None, 23 | level: TextType = '', 24 | path: t.Optional[str] = None, 25 | line_no: t.Optional[int] = None, 26 | link_path: t.Optional[str] = None, 27 | ) -> Text: 28 | output = Text(no_wrap=True) 29 | if self.show_time: 30 | log_time = log_time or console.get_datetime() 31 | time_format = time_format or self.time_format 32 | if callable(time_format): 33 | log_time_display = time_format(log_time) 34 | else: 35 | log_time_display = Text(log_time.strftime(time_format), style='log.time') 36 | if log_time_display == self._last_time and self.omit_repeated_times: 37 | output.append(' ' * len(log_time_display), style='log.time') 38 | else: 39 | output.append(log_time_display) 40 | self._last_time = log_time_display 41 | output.pad_right(1) 42 | 43 | if self.show_level: 44 | output.append(level) 45 | if self.level_width: 46 | output.pad_right(max(1, self.level_width - len(level))) 47 | else: 48 | output.pad_right(1) 49 | 50 | for renderable in Renderables(renderables): # type: ignore 51 | if isinstance(renderable, Text): 52 | renderable.stylize('log.message') 53 | 54 | output.append(renderable) 55 | output.pad_right(1) 56 | 57 | if self.show_path and path: 58 | path_text = Text(style='log.path') 59 | path_text.append(path, style=f'link file://{link_path}' if link_path else '') 60 | if line_no: 61 | path_text.append(':') 62 | path_text.append( 63 | f'{line_no}', 64 | style=f'link file://{link_path}#{line_no}' if link_path else '', 65 | ) 66 | output.append(path_text) 67 | output.pad_right(1) 68 | 69 | output.rstrip() 70 | return output 71 | 72 | 73 | def get_rich_log_handler(level: int = logging.WARNING, no_color: bool = False) -> RichHandler: 74 | console = get_console() 75 | console.soft_wrap = True 76 | console.no_color = no_color 77 | console.stderr = True 78 | 79 | handler = RichHandler( 80 | level, 81 | console, 82 | ) 83 | handler._log_render = _OneLineLogRender( 84 | show_level=True, 85 | show_path=False, 86 | omit_repeated_times=False, 87 | ) 88 | 89 | return handler 90 | 91 | 92 | def setup_logging(verbose: int = 0, log_file: t.Optional[str] = None, colored: bool = True) -> None: 93 | """ 94 | Setup logging stream handler 95 | 96 | :param verbose: 0 - WARNING, 1 - INFO, 2 - DEBUG 97 | :param log_file: log file path 98 | :param colored: colored output or not 99 | :return: None 100 | """ 101 | if not verbose: 102 | level = logging.WARNING 103 | elif verbose == 1: 104 | level = logging.INFO 105 | else: 106 | level = logging.DEBUG 107 | 108 | package_logger = logging.getLogger(__package__) 109 | package_logger.setLevel(level) 110 | 111 | if log_file: 112 | handler: logging.Handler = logging.FileHandler(log_file) 113 | else: 114 | handler = get_rich_log_handler(level, not colored) 115 | if package_logger.hasHandlers(): 116 | package_logger.handlers.clear() 117 | package_logger.addHandler(handler) 118 | 119 | package_logger.propagate = False # don't propagate to root logger 120 | -------------------------------------------------------------------------------- /idf_build_apps/manifest/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """ 5 | Manifest file 6 | """ 7 | 8 | from .manifest import DEFAULT_BUILD_TARGETS, FolderRule 9 | 10 | __all__ = [ 11 | 'DEFAULT_BUILD_TARGETS', 12 | 'FolderRule', 13 | ] 14 | 15 | from esp_bool_parser import register_addition_attribute 16 | 17 | 18 | def folder_rule_attr(target, **kwargs): 19 | return 1 if target in DEFAULT_BUILD_TARGETS.get() else 0 20 | 21 | 22 | register_addition_attribute('INCLUDE_DEFAULT', folder_rule_attr) 23 | -------------------------------------------------------------------------------- /idf_build_apps/manifest/soc_header.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import esp_bool_parser 5 | 6 | SOC_HEADERS = esp_bool_parser.SOC_HEADERS 7 | -------------------------------------------------------------------------------- /idf_build_apps/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/espressif/idf-build-apps/f83fc6ae5c0e0be0519aa2517bdaf728ba944838/idf_build_apps/py.typed -------------------------------------------------------------------------------- /idf_build_apps/session_args.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import os 6 | import re 7 | import typing as t 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | class SessionArgs: 13 | workdir: str = os.getcwd() 14 | override_sdkconfig_items: t.Dict[str, t.Any] = {} 15 | override_sdkconfig_file_path: t.Optional[str] = None 16 | 17 | def set(self, parsed_args, *, workdir=None): 18 | if workdir: 19 | self.workdir = workdir 20 | self._setup_override_sdkconfig(parsed_args) 21 | 22 | def clean(self): 23 | self.override_sdkconfig_items = {} 24 | self.override_sdkconfig_file_path = None 25 | 26 | def _setup_override_sdkconfig(self, args): 27 | override_sdkconfig_items = self._get_override_sdkconfig_items( 28 | args.override_sdkconfig_items.split(',') if args.override_sdkconfig_items else () 29 | ) 30 | override_sdkconfig_files_items = self._get_override_sdkconfig_files_items( 31 | args.override_sdkconfig_files.split(',') if args.override_sdkconfig_files else () 32 | ) 33 | 34 | override_sdkconfig_files_items.update(override_sdkconfig_items) 35 | self.override_sdkconfig_items = override_sdkconfig_files_items 36 | 37 | override_sdkconfig_merged_file = self._create_override_sdkconfig_merged_file(self.override_sdkconfig_items) 38 | self.override_sdkconfig_file_path = override_sdkconfig_merged_file 39 | 40 | def _get_override_sdkconfig_files_items(self, override_sdkconfig_files: t.Tuple[str]) -> t.Dict: 41 | d = {} 42 | for f in override_sdkconfig_files: 43 | # use filepath if abs/rel already point to itself 44 | if not os.path.isfile(f): 45 | # find it in the workdir 46 | LOGGER.debug('override sdkconfig file %s not found, checking under app_dir...', f) 47 | f = os.path.join(self.workdir, f) 48 | if not os.path.isfile(f): 49 | LOGGER.debug('override sdkconfig file %s not found, skipping...', f) 50 | continue 51 | 52 | with open(f) as fr: 53 | for line in fr: 54 | m = re.compile(r'^([^=]+)=([^\n]*)\n*$').match(line) 55 | if not m: 56 | continue 57 | d[m.group(1)] = m.group(2) 58 | return d 59 | 60 | def _get_override_sdkconfig_items(self, override_sdkconfig_items: t.Tuple[str]) -> t.Dict: 61 | d = {} 62 | for line in override_sdkconfig_items: 63 | m = re.compile(r'^([^=]+)=([^\n]*)\n*$').match(line) 64 | if m: 65 | d[m.group(1)] = m.group(2) 66 | return d 67 | 68 | def _create_override_sdkconfig_merged_file(self, override_sdkconfig_merged_items) -> t.Optional[str]: 69 | if not override_sdkconfig_merged_items: 70 | return None 71 | f_path = os.path.join(self.workdir, 'override-result.sdkconfig') 72 | with open(f_path, 'w+') as f: 73 | for key, value in override_sdkconfig_merged_items.items(): 74 | f.write(f'{key}={value}\n') 75 | return f_path 76 | -------------------------------------------------------------------------------- /idf_build_apps/utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import fnmatch 5 | import functools 6 | import glob 7 | import logging 8 | import os 9 | import shutil 10 | import subprocess 11 | import sys 12 | import typing as t 13 | from copy import ( 14 | deepcopy, 15 | ) 16 | from pathlib import Path 17 | 18 | from packaging.version import ( 19 | Version, 20 | ) 21 | from pydantic import BaseModel as _BaseModel 22 | 23 | LOGGER = logging.getLogger(__name__) 24 | 25 | if sys.version_info < (3, 8): 26 | from typing_extensions import ( 27 | Literal, 28 | ) 29 | else: 30 | from typing import ( 31 | Literal, # noqa 32 | ) 33 | 34 | if sys.version_info < (3, 11): 35 | from typing_extensions import ( 36 | Self, 37 | ) 38 | else: 39 | from typing import ( 40 | Self, # noqa 41 | ) 42 | 43 | 44 | class ConfigRule: 45 | def __init__(self, file_name: str, config_name: str = '') -> None: 46 | """ 47 | ConfigRule represents the sdkconfig file and the config name. 48 | 49 | For example: 50 | 51 | - filename='', config_name='default' - represents the default app configuration, and gives it a name 52 | 'default' 53 | - filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard 54 | value 55 | 56 | :param file_name: name of the sdkconfig file fragment, optionally with a single wildcard ('*' character). 57 | can also be empty to indicate that the default configuration of the app should be used 58 | :param config_name: name of the corresponding build configuration, or None if the value of wildcard is to be 59 | used 60 | """ 61 | self.file_name = file_name 62 | self.config_name = config_name 63 | 64 | 65 | def config_rules_from_str(rule_strings: t.Optional[t.List[str]]) -> t.List[ConfigRule]: 66 | """ 67 | Helper function to convert strings like 'file_name=config_name' into `ConfigRule` objects 68 | 69 | :param rule_strings: list of rules as strings or a single rule string 70 | :return: list of ConfigRules 71 | """ 72 | if not rule_strings: 73 | return [] 74 | 75 | rules = [] 76 | for rule_str in to_list(rule_strings): 77 | items = rule_str.split('=', 2) 78 | rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else '')) 79 | # '' is the default config, sort this one to the front 80 | return sorted(rules, key=lambda x: x.file_name) 81 | 82 | 83 | def get_parallel_start_stop(total: int, parallel_count: int, parallel_index: int) -> t.Tuple[int, int]: 84 | """ 85 | Calculate the start and stop indices for a parallel task (1-based). 86 | 87 | :param total: total number of tasks 88 | :param parallel_count: number of parallel tasks to run 89 | :param parallel_index: index of the parallel task to run 90 | :return: start and stop indices, [start, stop] 91 | """ 92 | if parallel_count == 1: 93 | return 1, total 94 | 95 | num_builds_per_job = (total + parallel_count - 1) // parallel_count 96 | 97 | _min = num_builds_per_job * (parallel_index - 1) + 1 98 | _max = min(num_builds_per_job * parallel_index, total) 99 | 100 | return _min, _max 101 | 102 | 103 | class BuildError(RuntimeError): 104 | pass 105 | 106 | 107 | class AutocompleteActivationError(SystemExit): 108 | pass 109 | 110 | 111 | class InvalidCommand(SystemExit): 112 | def __init__(self, msg: str) -> None: 113 | super().__init__('Invalid Command: ' + msg.strip()) 114 | 115 | 116 | class InvalidInput(SystemExit): 117 | """Invalid input from user""" 118 | 119 | 120 | class InvalidIfClause(SystemExit): 121 | """Invalid if clause in manifest file""" 122 | 123 | 124 | class InvalidManifest(SystemExit): 125 | """Invalid manifest file""" 126 | 127 | 128 | def rmdir(path: t.Union[Path, str], exclude_file_patterns: t.Union[t.List[str], str, None] = None) -> None: 129 | if not exclude_file_patterns: 130 | shutil.rmtree(path, ignore_errors=True) 131 | return 132 | 133 | for root, dirs, files in os.walk(path, topdown=False): 134 | for f in files: 135 | fp = os.path.join(root, f) 136 | remove = True 137 | for pattern in to_list(exclude_file_patterns): 138 | if pattern and fnmatch.fnmatch(f, pattern): 139 | remove = False 140 | break 141 | if remove: 142 | os.remove(fp) 143 | for d in dirs: 144 | try: 145 | os.rmdir(os.path.join(root, d)) 146 | except OSError: 147 | pass 148 | 149 | 150 | def find_first_match(pattern: str, path: str) -> t.Optional[str]: 151 | for root, _, files in os.walk(path): 152 | res = fnmatch.filter(files, pattern) 153 | if res: 154 | return os.path.join(root, res[0]) 155 | return None 156 | 157 | 158 | def subprocess_run( 159 | cmd: t.List[str], 160 | log_terminal: bool = True, 161 | log_fs: t.Union[t.IO[str], str, None] = None, 162 | check: bool = False, 163 | additional_env_dict: t.Optional[t.Dict[str, str]] = None, 164 | **kwargs, 165 | ) -> int: 166 | """ 167 | Subprocess.run for older python versions 168 | 169 | :param cmd: cmd 170 | :param log_terminal: print to `sys.stdout` if set to `True` 171 | :param log_fs: write to this file stream if not `None` 172 | :param check: raise `BuildError` when return code is non-zero 173 | :param additional_env_dict: additional environment variables 174 | :return: return code 175 | """ 176 | LOGGER.debug('==> Running %s', ' '.join(cmd)) 177 | 178 | subprocess_env = None 179 | if additional_env_dict is not None: 180 | subprocess_env = deepcopy(os.environ) 181 | subprocess_env.update(additional_env_dict) 182 | 183 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=subprocess_env, **kwargs) 184 | 185 | def _log_stdout(fs: t.Optional[t.IO[str]] = None): 186 | if p.stdout: 187 | for line in p.stdout: 188 | if isinstance(line, bytes): 189 | line = line.decode('utf-8') 190 | 191 | if log_terminal: 192 | sys.stdout.write(line) 193 | 194 | if fs: 195 | fs.write(line) 196 | 197 | if p.stdout: 198 | if log_fs: 199 | if isinstance(log_fs, str): 200 | with open(log_fs, 'a') as fa: 201 | _log_stdout(fa) 202 | else: 203 | _log_stdout(log_fs) 204 | 205 | returncode = p.wait() 206 | if check and returncode != 0: 207 | raise BuildError(f'Command {cmd} returned non-zero exit status {returncode}') 208 | 209 | return returncode 210 | 211 | 212 | _T = t.TypeVar('_T') 213 | 214 | 215 | @t.overload 216 | def to_list(s: None) -> None: ... 217 | 218 | 219 | @t.overload 220 | def to_list(s: t.Iterable[_T]) -> t.List[_T]: ... 221 | 222 | 223 | @t.overload 224 | def to_list(s: _T) -> t.List[_T]: ... 225 | 226 | 227 | def to_list(s): 228 | """ 229 | Turn all objects to lists 230 | 231 | :param s: anything 232 | :return: 233 | - ``None``, if ``s`` is None 234 | - itself, if ``s`` is a list 235 | - ``list(s)``, if ``s`` is a tuple or a set 236 | - ``[s]``, if ``s`` is other type 237 | 238 | """ 239 | if s is None: 240 | return s 241 | 242 | if isinstance(s, list): 243 | return s 244 | 245 | if isinstance(s, set) or isinstance(s, tuple): 246 | return list(s) 247 | 248 | return [s] 249 | 250 | 251 | @t.overload 252 | def to_set(s: None) -> None: ... 253 | 254 | 255 | @t.overload 256 | def to_set(s: t.Iterable[_T]) -> t.Set[_T]: ... 257 | 258 | 259 | @t.overload 260 | def to_set(s: _T) -> t.Set[_T]: ... 261 | 262 | 263 | def to_set(s): 264 | """ 265 | Turn all objects to sets 266 | 267 | :param s: anything 268 | :return: 269 | - ``None``, if ``s`` is None 270 | - itself, if ``s`` is a set 271 | - ``set(to_list(s))``, if ``s`` is other type 272 | """ 273 | if s is None: 274 | return s 275 | 276 | if isinstance(s, set): 277 | return s 278 | 279 | return set(to_list(s)) 280 | 281 | 282 | def semicolon_separated_str_to_list(s: t.Optional[str]) -> t.Optional[t.List[str]]: 283 | """ 284 | Split a string by semicolon and strip each part 285 | 286 | Args: 287 | s: string to split 288 | 289 | Returns: 290 | list of strings 291 | """ 292 | if s is None or s.strip() == '': 293 | return None 294 | 295 | return [p.strip() for p in s.strip().split(';') if p.strip()] 296 | 297 | 298 | def to_absolute_path(s: str, rootpath: t.Optional[str] = None) -> str: 299 | rp = os.path.abspath(os.path.expanduser(rootpath or '.')) 300 | 301 | sp = os.path.expanduser(s) 302 | if os.path.isabs(sp): 303 | return sp 304 | else: 305 | return os.path.abspath(os.path.join(rp, sp)) 306 | 307 | 308 | def to_version(s: t.Any) -> Version: 309 | if isinstance(s, Version): 310 | return s 311 | 312 | try: 313 | return Version(str(s)) 314 | except ValueError: 315 | raise InvalidInput(f'Invalid version: {s}') 316 | 317 | 318 | def files_matches_patterns( 319 | files: t.Union[t.List[str], str], 320 | patterns: t.Union[t.List[str], str], 321 | rootpath: t.Optional[str] = None, 322 | ) -> bool: 323 | # can't match an absolute pattern with a relative path 324 | # change all to absolute paths 325 | matched_paths = set() 326 | for pat in [to_absolute_path(p, rootpath) for p in to_list(patterns)]: 327 | matched_paths.update(glob.glob(str(pat), recursive=True)) 328 | 329 | for f in [to_absolute_path(f, rootpath) for f in to_list(files)]: 330 | if str(f) in matched_paths: 331 | return True 332 | 333 | return False 334 | 335 | 336 | @functools.total_ordering 337 | class BaseModel(_BaseModel): 338 | """ 339 | BaseModel that is hashable 340 | """ 341 | 342 | __EQ_IGNORE_FIELDS__: t.List[str] = [] 343 | __EQ_TUNE_FIELDS__: t.Dict[str, t.Callable[[t.Any], t.Any]] = {} 344 | 345 | def __lt__(self, other: t.Any) -> bool: 346 | if isinstance(other, self.__class__): 347 | for k in self.model_dump(): 348 | if k in self.__EQ_IGNORE_FIELDS__: 349 | continue 350 | 351 | self_attr = getattr(self, k, '') or '' 352 | other_attr = getattr(other, k, '') or '' 353 | 354 | if k in self.__EQ_TUNE_FIELDS__: 355 | self_attr = str(self.__EQ_TUNE_FIELDS__[k](self_attr)) 356 | other_attr = str(self.__EQ_TUNE_FIELDS__[k](other_attr)) 357 | 358 | if self_attr != other_attr: 359 | return self_attr < other_attr 360 | 361 | continue 362 | 363 | return False 364 | 365 | return NotImplemented 366 | 367 | def __eq__(self, other: t.Any) -> bool: 368 | if isinstance(other, self.__class__): 369 | # we only care the public attributes 370 | self_model_dump = self.model_dump() 371 | other_model_dump = other.model_dump() 372 | 373 | for _field in self.__EQ_IGNORE_FIELDS__: 374 | self_model_dump.pop(_field, None) 375 | other_model_dump.pop(_field, None) 376 | 377 | for _field in self.__EQ_TUNE_FIELDS__: 378 | self_model_dump[_field] = self.__EQ_TUNE_FIELDS__[_field](self_model_dump[_field]) 379 | other_model_dump[_field] = self.__EQ_TUNE_FIELDS__[_field](other_model_dump[_field]) 380 | 381 | return self_model_dump == other_model_dump 382 | 383 | return NotImplemented 384 | 385 | def __hash__(self) -> int: 386 | hash_list = [] 387 | 388 | self_model_dump = self.model_dump() 389 | for _field in self.__EQ_TUNE_FIELDS__: 390 | self_model_dump[_field] = self.__EQ_TUNE_FIELDS__[_field](self_model_dump[_field]) 391 | 392 | for v in self_model_dump.values(): 393 | if isinstance(v, list): 394 | hash_list.append(tuple(v)) 395 | elif isinstance(v, dict): 396 | hash_list.append(tuple(v.items())) 397 | else: 398 | hash_list.append(v) 399 | 400 | return hash((type(self), *tuple(hash_list))) 401 | 402 | 403 | def drop_none_kwargs(d: dict) -> dict: 404 | return {k: v for k, v in d.items() if v is not None} 405 | 406 | 407 | PathLike = t.Union[str, Path] 408 | -------------------------------------------------------------------------------- /idf_build_apps/vendors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/espressif/idf-build-apps/f83fc6ae5c0e0be0519aa2517bdaf728ba944838/idf_build_apps/vendors/__init__.py -------------------------------------------------------------------------------- /idf_build_apps/vendors/pydantic_sources.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Samuel Colvin and other contributors 2 | # SPDX-License-Identifier: MIT 3 | 4 | """ 5 | Partially copied from https://github.com/pydantic/pydantic-settings v2.5.2 6 | since python 3.7 version got dropped at pydantic-settings 2.1.0 7 | but the feature we need introduced in 2.2.0 8 | 9 | For contributing history please refer to the original github page 10 | For the full license text refer to 11 | https://github.com/pydantic/pydantic-settings/blob/9b73e924cab136d876907af0c6836dcca09ac35c/LICENSE 12 | 13 | Modifications: 14 | - use toml instead of tomli when python < 3.11 15 | - stop using global variables 16 | - fix some warnings 17 | - recursively find TOML file. 18 | """ 19 | 20 | import logging 21 | import os 22 | import sys 23 | from abc import ABC, abstractmethod 24 | from pathlib import Path 25 | from typing import Any, Dict, List, Optional, Tuple, Type, Union 26 | 27 | from pydantic_settings import InitSettingsSource 28 | from pydantic_settings.main import BaseSettings 29 | 30 | from idf_build_apps.constants import IDF_BUILD_APPS_TOML_FN 31 | 32 | PathType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] 33 | DEFAULT_PATH = Path('') 34 | LOGGER = logging.getLogger(__name__) 35 | 36 | 37 | class ConfigFileSourceMixin(ABC): 38 | def _read_files(self, files: Optional[PathType]) -> Dict[str, Any]: 39 | if files is None: 40 | return {} 41 | if isinstance(files, (str, os.PathLike)): 42 | files = [files] 43 | kwargs: Dict[str, Any] = {} 44 | for file in files: 45 | file_path = Path(file).expanduser() 46 | if file_path.is_file(): 47 | kwargs.update(self._read_file(file_path)) 48 | return kwargs 49 | 50 | @abstractmethod 51 | def _read_file(self, path: Optional[Path]) -> Dict[str, Any]: 52 | pass 53 | 54 | 55 | class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): 56 | """ 57 | A source class that loads variables from a TOML file 58 | """ 59 | 60 | def __init__( 61 | self, 62 | settings_cls: Type[BaseSettings], 63 | toml_file: Optional[Path] = DEFAULT_PATH, 64 | ): 65 | self.toml_file_path = self._pick_toml_file( 66 | toml_file, 67 | settings_cls.model_config.get('pyproject_toml_depth', sys.maxsize), 68 | IDF_BUILD_APPS_TOML_FN, 69 | ) 70 | self.toml_data = self._read_files(self.toml_file_path) 71 | super().__init__(settings_cls, self.toml_data) 72 | 73 | def _read_file(self, path: Optional[Path]) -> Dict[str, Any]: 74 | if not path or not path.is_file(): 75 | return {} 76 | 77 | if sys.version_info < (3, 11): 78 | import toml 79 | 80 | with open(path) as toml_file: 81 | return toml.load(toml_file) 82 | else: 83 | import tomllib 84 | 85 | with open(path, 'rb') as toml_file: 86 | return tomllib.load(toml_file) 87 | 88 | @staticmethod 89 | def _pick_toml_file(provided: Optional[Path], depth: int, filename: str) -> Optional[Path]: 90 | """ 91 | Pick a file path to use. If a file path is provided, use it. Otherwise, search up the directory tree for a 92 | file with the given name. 93 | 94 | :param provided: Explicit path provided when instantiating this class. 95 | :param depth: Number of directories up the tree to check of a pyproject.toml. 96 | """ 97 | if provided and Path(provided).is_file(): 98 | fp = provided.resolve() 99 | LOGGER.debug(f'Loading config file: {fp}') 100 | return fp 101 | 102 | rv = Path.cwd() 103 | count = -1 104 | while count < depth: 105 | if len(rv.parts) == 1: 106 | break 107 | 108 | fp = rv / filename 109 | if fp.is_file(): 110 | LOGGER.debug(f'Loading config file: {fp}') 111 | return fp 112 | 113 | rv = rv.parent 114 | count += 1 115 | 116 | return None 117 | 118 | 119 | class PyprojectTomlConfigSettingsSource(TomlConfigSettingsSource): 120 | """ 121 | A source class that loads variables from a `pyproject.toml` file. 122 | """ 123 | 124 | def __init__( 125 | self, 126 | settings_cls: Type[BaseSettings], 127 | toml_file: Optional[Path] = None, 128 | ) -> None: 129 | self.toml_file_path = self._pick_toml_file( 130 | toml_file, 131 | settings_cls.model_config.get('pyproject_toml_depth', sys.maxsize), 132 | 'pyproject.toml', 133 | ) 134 | self.toml_table_header: Tuple[str, ...] = settings_cls.model_config.get( 135 | 'pyproject_toml_table_header', 136 | ('tool', 'idf-build-apps'), 137 | ) 138 | self.toml_data = self._read_files(self.toml_file_path) 139 | for key in self.toml_table_header: 140 | self.toml_data = self.toml_data.get(key, {}) 141 | super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data) 142 | -------------------------------------------------------------------------------- /idf_build_apps/yaml/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from .parser import ( 5 | parse, 6 | ) 7 | 8 | __all__ = ['parse'] 9 | -------------------------------------------------------------------------------- /idf_build_apps/yaml/parser.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import typing as t 5 | 6 | import yaml 7 | 8 | from ..utils import PathLike 9 | 10 | 11 | def parse_postfixes(manifest_dict: t.Dict): 12 | for folder, folder_rule in manifest_dict.items(): 13 | if folder.startswith('.'): 14 | continue 15 | 16 | if not folder_rule: 17 | continue 18 | 19 | updated_folder: t.Dict = {} 20 | sorted_keys = sorted(folder_rule) 21 | for key in sorted_keys: 22 | if not key.endswith(('+', '-')): 23 | updated_folder[key] = folder_rule[key] 24 | continue 25 | 26 | operation = key[-1] 27 | 28 | if_dict_obj = [] 29 | other_dict_obj = [] 30 | str_obj = set() 31 | for obj in updated_folder[key[:-1]]: 32 | if isinstance(obj, t.Dict): 33 | if 'if' in obj: 34 | if_dict_obj.append(obj) 35 | else: 36 | other_dict_obj.append(obj) 37 | else: 38 | str_obj.add(obj) 39 | 40 | for obj in folder_rule[key]: 41 | if isinstance(obj, t.Dict): 42 | _l = obj['if'] 43 | if isinstance(_l, str): 44 | _l = _l.replace(' ', '') 45 | 46 | new_dict_obj = [] 47 | for obj_j in if_dict_obj: 48 | _r = obj_j['if'] 49 | if isinstance(_r, str): 50 | _r = _r.replace(' ', '') 51 | if _l != _r: 52 | new_dict_obj.append(obj_j) 53 | if_dict_obj = new_dict_obj 54 | 55 | if operation == '+': 56 | if_dict_obj.append(obj) 57 | else: 58 | str_obj.add(obj) if operation == '+' else str_obj.remove(obj) 59 | 60 | updated_folder[key[:-1]] = if_dict_obj + other_dict_obj + sorted(str_obj) 61 | 62 | manifest_dict[folder] = updated_folder 63 | 64 | 65 | def parse(path: PathLike) -> t.Dict: 66 | with open(path) as f: 67 | manifest_dict = yaml.safe_load(f) or {} 68 | parse_postfixes(manifest_dict) 69 | return manifest_dict 70 | -------------------------------------------------------------------------------- /license_header.txt: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD 2 | SPDX-License-Identifier: Apache-2.0 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "idf-build-apps" 7 | authors = [ 8 | {name = "Fu Hanxi", email = "fuhanxi@espressif.com"} 9 | ] 10 | readme = "README.md" 11 | license = {file = "LICENSE"} 12 | classifiers = [ 13 | "Development Status :: 5 - Production/Stable", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Programming Language :: Python :: 3.7", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Python :: 3.11", 20 | "Programming Language :: Python :: 3.12", 21 | "Programming Language :: Python :: 3.13", 22 | ] 23 | dynamic = ["version", "description"] 24 | requires-python = ">=3.7" 25 | 26 | dependencies = [ 27 | "pyparsing", 28 | "pyyaml", 29 | "packaging", 30 | "toml; python_version < '3.11'", 31 | "pydantic~=2.0", 32 | "pydantic_settings", 33 | "argcomplete>=3", 34 | "typing-extensions; python_version < '3.11'", 35 | "esp-bool-parser>=0.1.2,<1", 36 | # debug/print 37 | "rich", 38 | ] 39 | 40 | [project.optional-dependencies] 41 | test = [ 42 | "pytest", 43 | "pytest-cov", 44 | ] 45 | doc = [ 46 | "sphinx", 47 | # theme 48 | "sphinx-rtd-theme", 49 | # extensions 50 | "sphinx_copybutton", # copy button 51 | "myst-parser", # markdown support 52 | "sphinxcontrib-mermaid", # mermaid graph support 53 | "sphinx-argparse", # auto-generate cli help message 54 | "sphinx-tabs", # tabs 55 | ] 56 | 57 | [project.urls] 58 | homepage = "https://github.com/espressif/idf-build-apps" 59 | repository = "https://github.com/espressif/idf-build-apps" 60 | documentation = "https://docs.espressif.com/projects/idf-build-apps" 61 | changelog = "https://github.com/espressif/idf-build-apps/blob/main/CHANGELOG.md" 62 | 63 | [project.scripts] 64 | idf-build-apps = "idf_build_apps:main.main" 65 | 66 | [tool.commitizen] 67 | name = "cz_conventional_commits" 68 | version = "2.11.2" 69 | tag_format = "v$version" 70 | version_files = [ 71 | "idf_build_apps/__init__.py", 72 | ] 73 | 74 | [tool.pytest.ini_options] 75 | testpaths = [ 76 | "tests", 77 | ] 78 | 79 | [tool.isort] 80 | profile = 'black' 81 | force_grid_wrap = 1 82 | 83 | [tool.ruff] 84 | line-length = 120 85 | target-version = "py37" 86 | 87 | [tool.ruff.lint] 88 | select = [ 89 | 'F', # Pyflakes 90 | 'E', # pycodestyle 91 | 'W', # pycodestyle 92 | # 'C90', # mccabe 93 | 'I', # isort 94 | # 'N', # pep8-naming 95 | # 'D', # pydocstyle 96 | 'UP', # pyupgrade 97 | 'YTT', # flake8-2020 98 | # 'ANN', # flake8-annotations 99 | # 'ASYNC', # flake8-async 100 | # 'TRIO', # flake8-trio 101 | # 'S', # flake8-bandit 102 | # 'BLE', # flake8-blind-except 103 | # 'FBT', # flake8-boolean-trap 104 | # 'B', # flake8-bugbear 105 | 'A', # flake8-builtins 106 | # 'COM', # flake8-commas 107 | # 'CPY', # flake8-copyright 108 | # 'C4', # flake8-comprehensions 109 | # 'DTZ', # flake8-datetimez 110 | # 'T10', # flake8-debugger 111 | # 'DJ', # flake8-django 112 | # 'EM', # flake8-errmsg 113 | # 'EXE', # flake8-executable 114 | # 'FA', # flake8-future-annotations 115 | # 'ISC', # flake8-implicit-str-concat 116 | # 'ICN', # flake8-import-conventions 117 | # 'G', # flake8-logging-format 118 | # 'INP', # flake8-no-pep420 119 | # 'PIE', # flake8-pie 120 | # 'T20', # flake8-print 121 | # 'PYI', # flake8-pyi 122 | # 'PT', # flake8-pytest-style 123 | # 'Q', # flake8-quotes 124 | # 'RSE', # flake8-raise 125 | # 'RET', # flake8-return 126 | # 'SLF', # flake8-self 127 | # 'SLOT', # flake8-slots 128 | # 'SIM', # flake8-simplify 129 | # 'TID', # flake8-tidy-imports 130 | # 'TCH', # flake8-type-checking 131 | # 'INT', # flake8-gettext 132 | 'ARG', # flake8-unused-arguments 133 | # 'PTH', # flake8-use-pathlib 134 | 'TD', # flake8-todos 135 | 'FIX', # flake8-fixme 136 | 'ERA', # eradicate 137 | # 'PD', # pandas-vet 138 | # 'PGH', # pygrep-hooks 139 | # 'PL', # Pylint 140 | # 'TRY', # tryceratops 141 | # 'FLY', # flynt 142 | # 'NPY', # NumPy-specific rules 143 | # 'AIR', # Airflow 144 | # 'PERF', # Perflint 145 | 'FURB', # refurb 146 | 'LOG', # flake8-logging 147 | 'RUF', # Ruff-specific rules 148 | ] 149 | ignore = [ 150 | # Mutable class attributes should be annotated with `typing.ClassVar`, pydantic model is an exception 151 | 'RUF012', 152 | # `open` and `read` should be replaced by `Path(f).read_text()` 153 | 'FURB101', 154 | ] 155 | typing-modules = [ 156 | "idf_build_apps.utils" 157 | ] 158 | 159 | [tool.ruff.format] 160 | quote-style = "single" 161 | docstring-code-format = true 162 | 163 | [tool.ruff.lint.flake8-unused-arguments] 164 | ignore-variadic-names = true 165 | 166 | [tool.mypy] 167 | python_version = "3.8" 168 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | import os 4 | 5 | import pytest 6 | 7 | import idf_build_apps 8 | from idf_build_apps import ( 9 | App, 10 | setup_logging, 11 | ) 12 | from idf_build_apps.args import apply_config_file 13 | from idf_build_apps.manifest.manifest import FolderRule, reset_default_build_targets 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def clean_cls_attr(tmp_path): 18 | App.MANIFEST = None 19 | reset_default_build_targets() 20 | idf_build_apps.SESSION_ARGS.clean() 21 | apply_config_file(reset=True) 22 | os.chdir(tmp_path) 23 | 24 | 25 | @pytest.fixture(autouse=True) 26 | def setup_logging_debug(): 27 | setup_logging(1) 28 | 29 | 30 | def create_project(name, folder): 31 | p = str(folder / name) 32 | os.makedirs(p) 33 | os.makedirs(os.path.join(p, 'main')) 34 | 35 | with open(os.path.join(p, 'CMakeLists.txt'), 'w') as fw: 36 | fw.write( 37 | f"""cmake_minimum_required(VERSION 3.16) 38 | include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) 39 | project({name}) 40 | """ 41 | ) 42 | 43 | with open(os.path.join(p, 'main', 'CMakeLists.txt'), 'w') as fw: 44 | fw.write( 45 | f"""idf_component_register(SRCS "{name}.c" 46 | INCLUDE_DIRS ".") 47 | """ 48 | ) 49 | 50 | with open(os.path.join(p, 'main', f'{name}.c'), 'w') as fw: 51 | fw.write( 52 | """#include 53 | void app_main(void) {} 54 | """ 55 | ) 56 | 57 | 58 | @pytest.fixture 59 | def sha_of_enable_only_esp32(): 60 | sha = FolderRule('test1', enable=[{'if': 'IDF_TARGET == "esp32"'}]).sha 61 | 62 | # !!! ONLY CHANGE IT WHEN NECESSARY !!! 63 | assert ( 64 | sha 65 | == 'bfc1c61176cfb76169eab6c4f00dbcc4d7886fee4b93ede5fac2c005dd85db640464e9b03986d3da3bfaa4d109b053862c07dc4d5a407e58f773a8f710ec60cb' # noqa: E501 66 | ) 67 | 68 | return sha 69 | 70 | 71 | @pytest.fixture 72 | def sha_of_enable_esp32_or_esp32s2(): 73 | sha = FolderRule('test1', enable=[{'if': 'IDF_TARGET == "esp32" or IDF_TARGET == "esp32s2"'}]).sha 74 | 75 | # !!! ONLY CHANGE IT WHEN NECESSARY !!! 76 | assert ( 77 | sha 78 | == '9ab121a0d39bcb590465837091e82dfd798cd1ff9579e92c23e8bebaee127b46751108f0de3953993cb7993903e45d78851fc465d774a606b0ab1251fbe4b9f5' # noqa: E501 79 | ) 80 | 81 | return sha 82 | -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | import contextlib 4 | import os 5 | 6 | import pytest 7 | from pydantic import ( 8 | ValidationError, 9 | ) 10 | 11 | from idf_build_apps import ( 12 | AppDeserializer, 13 | CMakeApp, 14 | MakeApp, 15 | ) 16 | from idf_build_apps.main import ( 17 | json_to_app, 18 | ) 19 | from idf_build_apps.utils import Literal 20 | 21 | 22 | def test_serialization(): 23 | a = CMakeApp('foo', 'bar') 24 | a_s = a.to_json() 25 | 26 | b = CMakeApp.model_validate_json(a_s) 27 | assert a == b 28 | 29 | 30 | def test_deserialization(): 31 | a = CMakeApp('foo', 'bar', size_json_filename='size.json') 32 | b = MakeApp('foo', 'bar', build_log_filename='build.log') 33 | 34 | assert a != b 35 | 36 | with open('test.txt', 'w') as fw: 37 | fw.write(a.to_json() + '\n') 38 | fw.write(b.to_json() + '\n') 39 | 40 | with open('test.txt') as fr: 41 | a_s = AppDeserializer.from_json(fr.readline()) 42 | b_s = AppDeserializer.from_json(fr.readline()) 43 | 44 | assert a == a_s 45 | assert b == b_s 46 | 47 | 48 | def test_app_sorting(): 49 | a = CMakeApp('foo', 'esp32') 50 | b = MakeApp('foo', 'esp32') 51 | 52 | c = CMakeApp('foo', 'esp32', size_json_filename='size.json') 53 | d = CMakeApp('foo', 'esp32s2') 54 | e = CMakeApp('foo', 'esp32s2', build_comment='build_comment') 55 | 56 | with pytest.raises(TypeError, match="'<' not supported between instances of 'CMakeApp' and 'MakeApp'"): 57 | assert a < b 58 | 59 | assert a < c < d 60 | assert d > c > a 61 | 62 | # __EQ_IGNORE_FIELDS__ 63 | assert d == e 64 | assert not d < e 65 | assert not d > e 66 | 67 | 68 | def test_app_deserializer(): 69 | a = CMakeApp('foo', 'esp32') 70 | b = MakeApp('foo', 'esp32') 71 | 72 | class CustomApp(CMakeApp): 73 | build_system: Literal['custom'] = 'custom' # type: ignore 74 | 75 | c = CustomApp('foo', 'esp32') 76 | 77 | assert json_to_app(a.to_json()) == a 78 | assert json_to_app(b.to_json()) == b 79 | 80 | with pytest.raises( 81 | ValidationError, 82 | match="Input tag 'custom' found using 'build_system' does not match any of the expected tags: 'cmake', 'make'", 83 | ): 84 | assert json_to_app(c.to_json()) == c 85 | 86 | assert json_to_app(c.to_json(), extra_classes=[CustomApp]) == c 87 | 88 | 89 | def test_app_init_from_another(): 90 | a = CMakeApp('foo', 'esp32', build_dir='build_@t_') 91 | b = CMakeApp.from_another(a, target='esp32c3') 92 | 93 | assert isinstance(b, CMakeApp) 94 | assert a.target == 'esp32' 95 | assert b.target == 'esp32c3' 96 | assert 'build_esp32_' == a.build_dir 97 | assert 'build_esp32c3_' == b.build_dir 98 | 99 | 100 | def test_app_hash(): 101 | a = CMakeApp('foo', 'esp32') 102 | b = CMakeApp('foo/', 'esp32') 103 | assert a == b 104 | assert hash(a) == hash(b) 105 | assert len(list({a, b})) == 1 106 | 107 | with contextlib.chdir(os.path.expanduser('~')): 108 | a = CMakeApp('foo', 'esp32') 109 | b = CMakeApp(os.path.join('~', 'foo'), 'esp32') 110 | assert a == b 111 | assert hash(a) == hash(b) 112 | assert len(list({a, b})) == 1 113 | -------------------------------------------------------------------------------- /tests/test_build.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import shutil 6 | import subprocess 7 | from copy import ( 8 | deepcopy, 9 | ) 10 | from pathlib import Path 11 | from xml.etree import ( 12 | ElementTree, 13 | ) 14 | 15 | import pytest 16 | from conftest import ( 17 | create_project, 18 | ) 19 | 20 | from idf_build_apps import ( 21 | build_apps, 22 | find_apps, 23 | ) 24 | from idf_build_apps.app import ( 25 | App, 26 | CMakeApp, 27 | ) 28 | from idf_build_apps.args import BuildArguments 29 | from idf_build_apps.constants import ( 30 | IDF_PATH, 31 | BuildStatus, 32 | ) 33 | from idf_build_apps.utils import Literal 34 | 35 | 36 | @pytest.mark.skipif(not shutil.which('idf.py'), reason='idf.py not found') 37 | class TestBuild: 38 | def test_build_hello_world(self, tmp_path, capsys): 39 | path = os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world') 40 | 41 | app = CMakeApp(path, 'esp32', work_dir=str(tmp_path / 'test')) 42 | app.build() 43 | 44 | captured = capsys.readouterr() 45 | assert 'Configuring done' in captured.out 46 | assert 'Project build complete.' in captured.out 47 | assert app.build_status == BuildStatus.SUCCESS 48 | 49 | @pytest.mark.parametrize( 50 | 'modified_components, check_app_dependencies, build_status', 51 | [ 52 | (None, True, BuildStatus.SUCCESS), 53 | ([], True, BuildStatus.SKIPPED), 54 | ([], False, BuildStatus.SUCCESS), 55 | ('fake', True, BuildStatus.SKIPPED), 56 | ('fake', False, BuildStatus.SUCCESS), 57 | ('soc', True, BuildStatus.SUCCESS), 58 | ('soc', False, BuildStatus.SUCCESS), 59 | (['soc', 'fake'], True, BuildStatus.SUCCESS), 60 | ], 61 | ) 62 | def test_build_with_modified_components(self, tmp_path, modified_components, check_app_dependencies, build_status): 63 | path = os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world') 64 | 65 | app = CMakeApp(path, 'esp32', work_dir=str(tmp_path / 'test')) 66 | app.build( 67 | modified_components=modified_components, 68 | check_app_dependencies=check_app_dependencies, 69 | ) 70 | assert app.build_status == build_status 71 | 72 | @pytest.mark.parametrize( 73 | 'modified_files, build_status', 74 | [ 75 | ('/foo', BuildStatus.SKIPPED), 76 | (os.path.join(IDF_PATH, 'examples', 'README.md'), BuildStatus.SKIPPED), 77 | ([os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world', 'README.md')], BuildStatus.SKIPPED), 78 | ( 79 | [ 80 | os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world', 'README.md'), 81 | os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world', 'main', 'hello_world_main.c'), 82 | ], 83 | BuildStatus.SUCCESS, 84 | ), 85 | ], 86 | ) 87 | def test_build_with_modified_files(self, modified_files, build_status): 88 | test_dir = os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world') 89 | 90 | app = CMakeApp(test_dir, 'esp32') 91 | app.build( 92 | modified_components=[], 93 | modified_files=modified_files, 94 | check_app_dependencies=True, 95 | ) 96 | 97 | assert app.build_status == build_status 98 | 99 | def test_build_without_modified_components_but_ignored_app_dependency_check(self): 100 | test_dir = os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world') 101 | 102 | apps = find_apps( 103 | test_dir, 104 | 'esp32', 105 | modified_components=[], 106 | modified_files=['foo.c'], 107 | ignore_app_dependencies_filepatterns=['foo.c'], 108 | ) 109 | 110 | for app in apps: 111 | app.build() 112 | assert app.build_status == BuildStatus.SUCCESS 113 | 114 | def test_build_with_junit_output(self, tmp_path): 115 | test_dir = os.path.join(IDF_PATH, 'examples', 'get-started', 'hello_world') 116 | 117 | apps = [ 118 | CMakeApp(test_dir, 'esp32', build_dir='build_1'), 119 | CMakeApp(test_dir, 'esp32', build_dir='build_2'), 120 | CMakeApp(test_dir, 'esp32', build_dir='build_3'), 121 | CMakeApp(test_dir, 'esp32', build_dir='build_4'), 122 | ] 123 | apps[2].build_status = BuildStatus.DISABLED 124 | apps[3].build_status = BuildStatus.SKIPPED 125 | 126 | build_apps(deepcopy(apps), dry_run=True, junitxml=str(tmp_path / 'test.xml')) 127 | 128 | with open('test.xml') as f: 129 | xml = ElementTree.fromstring(f.read()) 130 | 131 | test_suite = xml.findall('testsuite')[0] 132 | assert test_suite.attrib['tests'] == '0' 133 | assert test_suite.attrib['failures'] == '0' 134 | assert test_suite.attrib['errors'] == '0' 135 | assert test_suite.attrib['skipped'] == '4' 136 | 137 | for i, testcase in enumerate(test_suite.findall('testcase')): 138 | assert testcase.attrib['name'] == apps[i].build_path 139 | assert float(testcase.attrib['time']) > 0 140 | assert testcase.find('skipped') is not None 141 | if i in (0, 1): 142 | assert testcase.find('skipped').attrib['message'] == 'dry run' 143 | elif i == 2: 144 | assert testcase.find('skipped').attrib['message'] == 'Build disabled. Skipping...' 145 | elif i == 3: 146 | assert testcase.find('skipped').attrib['message'] == 'Build skipped. Skipping...' 147 | else: 148 | assert False # not expected 149 | 150 | build_apps(deepcopy(apps), junitxml=str(tmp_path / 'test.xml')) 151 | 152 | with open('test.xml') as f: 153 | xml = ElementTree.fromstring(f.read()) 154 | 155 | test_suite = xml.findall('testsuite')[0] 156 | assert test_suite.attrib['tests'] == '2' 157 | assert test_suite.attrib['failures'] == '0' 158 | assert test_suite.attrib['errors'] == '0' 159 | assert test_suite.attrib['skipped'] == '2' 160 | 161 | for i, testcase in enumerate(test_suite.findall('testcase')): 162 | assert float(testcase.attrib['time']) > 0 163 | assert testcase.attrib['name'] == apps[i].build_path 164 | assert testcase.find('error') is None 165 | assert testcase.find('failure') is None 166 | if i in (0, 1): 167 | assert testcase.find('skipped') is None 168 | elif i == 2: 169 | assert testcase.find('skipped').attrib['message'] == 'Build disabled. Skipping...' 170 | elif i == 3: 171 | assert testcase.find('skipped').attrib['message'] == 'Build skipped. Skipping...' 172 | else: 173 | assert False # not expected 174 | 175 | def test_work_dir_inside_relative_app_dir(self, tmp_path): 176 | create_project('foo', tmp_path) 177 | 178 | os.chdir(tmp_path / 'foo') 179 | apps = find_apps( 180 | '.', 181 | 'esp32', 182 | work_dir=os.path.join('foo', 'bar'), 183 | ) 184 | build_apps(apps) 185 | 186 | assert len(apps) == 1 187 | assert apps[0].build_status == BuildStatus.SUCCESS 188 | 189 | def test_build_apps_without_passing_apps(self, tmp_path): 190 | create_project('foo', tmp_path) 191 | 192 | os.chdir(tmp_path / 'foo') 193 | ret_code = build_apps( 194 | target='esp32', 195 | work_dir=os.path.join('foo', 'bar'), 196 | junitxml='test.xml', 197 | ) 198 | assert ret_code == 0 199 | 200 | with open('test.xml') as f: 201 | xml = ElementTree.fromstring(f.read()) 202 | 203 | test_suite = xml.findall('testsuite')[0] 204 | assert test_suite.attrib['tests'] == '1' 205 | assert test_suite.attrib['failures'] == '0' 206 | assert test_suite.attrib['errors'] == '0' 207 | assert test_suite.attrib['skipped'] == '0' 208 | 209 | assert test_suite.findall('testcase')[0].attrib['name'] == 'foo/bar/build' 210 | 211 | 212 | class CustomClassApp(App): 213 | build_system: Literal['custom_class'] = 'custom_class' # type: ignore 214 | 215 | def build(self, *args, **kwargs): 216 | # For testing, we'll just create a dummy build directory 217 | if not self.dry_run: 218 | os.makedirs(self.build_path, exist_ok=True) 219 | with open(os.path.join(self.build_path, 'dummy.txt'), 'w') as f: 220 | f.write('Custom build successful') 221 | self.build_status = BuildStatus.SUCCESS 222 | print('Custom build successful') 223 | 224 | @classmethod 225 | def is_app(cls, path: str) -> bool: # noqa: ARG003 226 | return True 227 | 228 | 229 | @pytest.mark.skipif(not shutil.which('idf.py'), reason='idf.py not found') 230 | class TestBuildWithCustomApp: 231 | custom_app_code = """ 232 | from idf_build_apps import App 233 | import os 234 | from idf_build_apps.constants import BuildStatus 235 | from idf_build_apps.utils import Literal 236 | 237 | class CustomApp(App): 238 | build_system: Literal['custom'] = 'custom' 239 | 240 | def build(self, *args, **kwargs): 241 | # For testing, we'll just create a dummy build directory 242 | if not self.dry_run: 243 | os.makedirs(self.build_path, exist_ok=True) 244 | with open(os.path.join(self.build_path, 'dummy.txt'), 'w') as f: 245 | f.write('Custom build successful') 246 | self.build_status = BuildStatus.SUCCESS 247 | print('Custom build successful') 248 | 249 | @classmethod 250 | def is_app(cls, path: str) -> bool: 251 | return True 252 | """ 253 | 254 | @pytest.fixture(autouse=True) 255 | def _setup(self, tmp_path: Path, monkeypatch): 256 | os.chdir(tmp_path) 257 | 258 | test_app = tmp_path / 'test_app' 259 | 260 | test_app.mkdir() 261 | (test_app / 'main' / 'main.c').parent.mkdir(parents=True) 262 | (test_app / 'main' / 'main.c').write_text('void app_main() {}') 263 | 264 | # Create a custom app module 265 | custom_module = tmp_path / 'custom.py' 266 | custom_module.write_text(self.custom_app_code) 267 | 268 | monkeypatch.setenv('PYTHONPATH', os.getenv('PYTHONPATH', '') + os.pathsep + str(tmp_path)) 269 | 270 | return test_app 271 | 272 | def test_custom_app_cli(self, tmp_path): 273 | subprocess.run( 274 | [ 275 | 'idf-build-apps', 276 | 'build', 277 | '-p', 278 | 'test_app', 279 | '--target', 280 | 'esp32', 281 | '--build-system', 282 | 'custom:CustomApp', 283 | ], 284 | check=True, 285 | ) 286 | 287 | assert (tmp_path / 'test_app' / 'build' / 'dummy.txt').exists() 288 | assert (tmp_path / 'test_app' / 'build' / 'dummy.txt').read_text() == 'Custom build successful' 289 | 290 | def test_custom_app_function(self, tmp_path): 291 | # Import the custom app class 292 | # Find and build the app using the imported CustomApp class 293 | apps = find_apps( 294 | paths=['test_app'], 295 | target='esp32', 296 | build_system=CustomClassApp, 297 | ) 298 | 299 | assert len(apps) == 1 300 | app = apps[0] 301 | assert isinstance(app, CustomClassApp) 302 | assert app.build_system == 'custom_class' 303 | 304 | # Build the app 305 | app.build() 306 | assert app.build_status == BuildStatus.SUCCESS 307 | assert (tmp_path / 'test_app' / 'build' / 'dummy.txt').exists() 308 | assert (tmp_path / 'test_app' / 'build' / 'dummy.txt').read_text() == 'Custom build successful' 309 | 310 | 311 | def test_build_apps_collect_files_when_no_apps_built(tmp_path): 312 | os.chdir(tmp_path) 313 | 314 | build_apps( 315 | build_arguments=BuildArguments( 316 | target='esp32', 317 | collect_app_info_filename='app_info.txt', 318 | collect_size_info_filename='size_info.txt', 319 | ) 320 | ) 321 | 322 | assert os.path.exists('app_info.txt') 323 | assert os.path.exists('size_info.txt') 324 | -------------------------------------------------------------------------------- /tests/test_cmd.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | import os 4 | import sys 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | from idf_build_apps.main import main 10 | from idf_build_apps.utils import InvalidCommand 11 | 12 | 13 | @pytest.mark.parametrize( 14 | 'args, expected_error', 15 | [ 16 | ([], 'the following arguments are required: --manifest-files, --output/-o'), 17 | (['--output', 'test.sha1'], 'the following arguments are required: --manifest-files'), 18 | (['--manifest-files', 'test.yml'], 'the following arguments are required: --output'), 19 | (['--manifest-files', 'test.yml', '--output', 'test.sha1'], None), 20 | ], 21 | ) 22 | def test_manifest_dump_sha_values( 23 | args, expected_error, sha_of_enable_only_esp32, sha_of_enable_esp32_or_esp32s2, capsys, monkeypatch 24 | ): 25 | Path('test.yml').write_text( 26 | """ 27 | foo: 28 | enable: 29 | - if: IDF_TARGET == "esp32" 30 | bar: 31 | enable: 32 | - if: IDF_TARGET == "esp32" 33 | baz: 34 | enable: 35 | - if: IDF_TARGET == "esp32" 36 | foobar: 37 | enable: 38 | - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32s2" 39 | """, 40 | encoding='utf8', 41 | ) 42 | 43 | try: 44 | with monkeypatch.context() as m: 45 | m.setattr(sys, 'argv', ['idf-build-apps', 'dump-manifest-sha', *args]) 46 | 47 | if not expected_error: 48 | main() 49 | else: 50 | main() 51 | except SystemExit as e: 52 | if isinstance(e, InvalidCommand): 53 | actual_error = e.args[0] 54 | else: 55 | actual_error = capsys.readouterr().err 56 | else: 57 | actual_error = None 58 | 59 | if expected_error: 60 | assert actual_error is not None 61 | assert expected_error in str(actual_error) 62 | else: 63 | with open('test.sha1') as fr: 64 | assert fr.read() == ( 65 | f'bar:{sha_of_enable_only_esp32}\n' 66 | f'baz:{sha_of_enable_only_esp32}\n' 67 | f'foo:{sha_of_enable_only_esp32}\n' 68 | f'foobar:{sha_of_enable_esp32_or_esp32s2}\n' 69 | ) 70 | 71 | 72 | def test_manifest_patterns(tmp_path, monkeypatch, capsys): 73 | manifest = tmp_path / 'manifest.yml' 74 | manifest.write_text( 75 | """foo: 76 | enable: 77 | - if: IDF_TARGET == "esp32" 78 | bar: 79 | enable: 80 | - if: IDF_TARGET == "esp32" 81 | """ 82 | ) 83 | 84 | with pytest.raises(SystemExit) as e: 85 | with monkeypatch.context() as m: 86 | m.setattr( 87 | sys, 88 | 'argv', 89 | ['idf-build-apps', 'find', '--manifest-filepatterns', '**.whatever', '**/manifest.yml', '-vv'], 90 | ) 91 | main() 92 | assert e.retcode == 0 93 | 94 | _, err = capsys.readouterr() 95 | assert f'Loading manifest file {os.path.join(tmp_path, "manifest.yml")}' in err 96 | assert f'"{os.path.join(tmp_path, "foo")}" does not exist' in err 97 | assert f'"{os.path.join(tmp_path, "bar")}" does not exist' in err 98 | -------------------------------------------------------------------------------- /tests/test_manifest.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | import os 4 | 5 | import esp_bool_parser 6 | import pytest 7 | import yaml 8 | from esp_bool_parser.utils import to_version 9 | 10 | import idf_build_apps 11 | from idf_build_apps import setup_logging 12 | from idf_build_apps.constants import ( 13 | SUPPORTED_TARGETS, 14 | ) 15 | from idf_build_apps.manifest.manifest import ( 16 | IfClause, 17 | Manifest, 18 | ) 19 | from idf_build_apps.utils import ( 20 | InvalidIfClause, 21 | InvalidManifest, 22 | ) 23 | from idf_build_apps.yaml import ( 24 | parse, 25 | ) 26 | 27 | 28 | class TestManifest: 29 | @pytest.fixture(autouse=True) 30 | def _setup_version(self, monkeypatch): 31 | """Setup ESP-IDF version for all tests in this class.""" 32 | monkeypatch.setattr(esp_bool_parser.constants, 'IDF_VERSION', to_version('5.9.0')) 33 | monkeypatch.setattr(esp_bool_parser.constants, 'IDF_VERSION_MAJOR', 5) 34 | monkeypatch.setattr(esp_bool_parser.constants, 'IDF_VERSION_MINOR', 9) 35 | monkeypatch.setattr(esp_bool_parser.constants, 'IDF_VERSION_PATCH', 0) 36 | 37 | def test_manifest_from_file_warning(self, tmp_path, capsys, monkeypatch): 38 | setup_logging() 39 | yaml_file = tmp_path / 'test.yml' 40 | yaml_file.write_text( 41 | """ 42 | test1: 43 | enable: 44 | - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32c3" 45 | - if: IDF_TARGET == "esp32s2" and IDF_VERSION_MAJOR in [4, 5] 46 | disable_test: 47 | - if: IDF_TARGET == "esp32c3" 48 | 49 | test2: 50 | enable: 51 | - if: INCLUDE_DEFAULT == 0 and IDF_TARGET == "linux" 52 | """, 53 | encoding='utf8', 54 | ) 55 | 56 | manifest = Manifest.from_file(yaml_file, root_path=tmp_path) 57 | captured_err = capsys.readouterr().err.splitlines() 58 | msg_fmt = 'Folder "{}" does not exist. Please check your manifest file {}' 59 | # two warnings warn test1 test2 not exists 60 | assert len(captured_err) == 2 61 | assert msg_fmt.format(os.path.join(tmp_path, 'test1'), yaml_file) in captured_err[0] 62 | assert msg_fmt.format(os.path.join(tmp_path, 'test2'), yaml_file) in captured_err[1] 63 | 64 | assert manifest.enable_build_targets('test1') == ['esp32', 'esp32c3', 'esp32s2'] 65 | assert manifest.enable_test_targets('test1') == ['esp32', 'esp32s2'] 66 | assert manifest.enable_build_targets('test2') == ['linux'] 67 | assert manifest.enable_test_targets('test2') == ['linux'] 68 | 69 | monkeypatch.setattr(idf_build_apps.manifest.manifest.Manifest, 'CHECK_MANIFEST_RULES', True) 70 | with pytest.raises(InvalidManifest, match=msg_fmt.format(os.path.join(tmp_path, 'test1'), yaml_file)): 71 | Manifest.from_file(yaml_file, root_path=tmp_path) 72 | 73 | # test with folder that has the same prefix as one of the folders in the manifest 74 | assert manifest.enable_build_targets('test23') == sorted(SUPPORTED_TARGETS) 75 | 76 | def test_repr(self, tmp_path): 77 | yaml_file = tmp_path / 'test.yml' 78 | 79 | yaml_file.write_text( 80 | """ 81 | test1: 82 | enable: 83 | - if: IDF_TARGET == "esp32c3" 84 | reason: "None" 85 | depends_components: 86 | - if: IDF_VERSION == "1.2.3" or IDF_VERSION_MAJOR == 4 87 | content: [ "VVV" ] 88 | - if: CONFIG_NAME == "AAA" 89 | content: [ "AAA" ] 90 | - default: ["some_1", "some_2", "some_3"] 91 | 92 | """, 93 | encoding='utf8', 94 | ) 95 | 96 | manifest = Manifest.from_file(yaml_file) 97 | manifest_rule = manifest.rules[0] 98 | assert ( 99 | repr(manifest_rule.enable) 100 | == """[IfClause(stmt='IDF_TARGET == "esp32c3"', temporary=False, reason='None')]""" 101 | ) 102 | assert ( 103 | repr(manifest_rule.depends_components) 104 | == """SwitchClause(if_clauses=[IfClause(stmt='IDF_VERSION == "1.2.3" or IDF_VERSION_MAJOR == 4', temporary=False, reason=None), IfClause(stmt='CONFIG_NAME == "AAA"', temporary=False, reason=None)], contents=[['VVV'], ['AAA']], default_clause=['some_1', 'some_2', 'some_3'])""" # noqa 105 | ) 106 | 107 | def test_manifest_switch_clause(self, tmp_path): 108 | yaml_file = tmp_path / 'test.yml' 109 | yaml_file.write_text( 110 | """ 111 | test1: 112 | depends_components: 113 | - if: IDF_VERSION == "5.9.0" 114 | content: [ "VVV" ] 115 | - if: CONFIG_NAME == "AAA" 116 | content: [ "AAA" ] 117 | - default: ["some_1", "some_2", "some_3"] 118 | 119 | test2: 120 | depends_components: 121 | - if: IDF_TARGET == "esp32" 122 | content: [ "esp32" ] 123 | - if: CONFIG_NAME == "AAA" 124 | content: [ "AAA" ] 125 | - if: IDF_VERSION == "5.9.0" 126 | content: [ "VVV" ] 127 | - default: ["some_1", "some_2", "some_3"] 128 | 129 | test3: 130 | depends_components: 131 | - if: CONFIG_NAME == "AAA" 132 | content: [ "AAA" ] 133 | - if: CONFIG_NAME == "BBB" 134 | content: [ "BBB" ] 135 | - default: ["some_1", "some_2", "some_3"] 136 | 137 | test4: 138 | depends_components: 139 | - if: CONFIG_NAME == "BBB" 140 | content: [ "BBB" ] 141 | - if: CONFIG_NAME == "AAA" 142 | content: [ "AAA" ] 143 | 144 | test5: 145 | depends_components: 146 | - "some_1" 147 | - "some_2" 148 | - "some_3" 149 | 150 | """, 151 | encoding='utf8', 152 | ) 153 | 154 | manifest = Manifest.from_file(yaml_file) 155 | 156 | assert manifest.depends_components('test1', None, None) == ['VVV'] 157 | assert manifest.depends_components('test1', None, 'AAA') == ['VVV'] 158 | 159 | assert manifest.depends_components('test2', 'esp32', None) == ['esp32'] 160 | assert manifest.depends_components('test2', None, 'AAA') == ['AAA'] 161 | assert manifest.depends_components('test2', 'esp32', 'AAA') == ['esp32'] 162 | assert manifest.depends_components('test2', None, None) == ['VVV'] 163 | 164 | assert manifest.depends_components('test3', 'esp32', 'AAA') == ['AAA'] 165 | assert manifest.depends_components('test3', 'esp32', 'BBB') == ['BBB'] 166 | assert manifest.depends_components('test3', 'esp32', '123123') == ['some_1', 'some_2', 'some_3'] 167 | assert manifest.depends_components('test3', None, None) == ['some_1', 'some_2', 'some_3'] 168 | 169 | assert manifest.depends_components('test4', 'esp32', 'AAA') == ['AAA'] 170 | assert manifest.depends_components('test4', 'esp32', 'BBB') == ['BBB'] 171 | assert manifest.depends_components('test4', 'esp32', '123123') == [] 172 | assert manifest.depends_components('test4', None, None) == [] 173 | 174 | assert manifest.depends_components('test5', 'esp32', 'AAA') == ['some_1', 'some_2', 'some_3'] 175 | assert manifest.depends_components('test5', 'esp32', 'BBB') == ['some_1', 'some_2', 'some_3'] 176 | assert manifest.depends_components('test5', 'esp32', '123123') == ['some_1', 'some_2', 'some_3'] 177 | assert manifest.depends_components('test5', None, None) == ['some_1', 'some_2', 'some_3'] 178 | 179 | def test_manifest_switch_clause_with_postfix(self, tmp_path): 180 | yaml_file = tmp_path / 'test.yml' 181 | 182 | yaml_file.write_text( 183 | """ 184 | .test: &test 185 | depends_components+: 186 | - if: CONFIG_NAME == "AAA" 187 | content: ["NEW_AAA"] 188 | - if: CONFIG_NAME == "BBB" 189 | content: ["NEW_BBB"] 190 | depends_components-: 191 | - if: CONFIG_NAME == "CCC" 192 | 193 | test1: 194 | <<: *test 195 | depends_components: 196 | - if: CONFIG_NAME == "AAA" 197 | content: [ "AAA" ] 198 | - if: CONFIG_NAME == "CCC" 199 | content: [ "CCC" ] 200 | - default: ["DF"] 201 | """, 202 | encoding='utf8', 203 | ) 204 | manifest = Manifest.from_file(yaml_file, root_path=tmp_path) 205 | 206 | assert manifest.depends_components('test1', None, None) == ['DF'] 207 | assert manifest.depends_components('test1', None, 'CCC') == ['DF'] 208 | assert manifest.depends_components('test1', None, 'AAA') == ['NEW_AAA'] 209 | assert manifest.depends_components('test1', None, 'BBB') == ['NEW_BBB'] 210 | 211 | def test_manifest_switch_clause_wrong_manifest_format(self, tmp_path): 212 | yaml_file = tmp_path / 'test.yml' 213 | 214 | yaml_file.write_text( 215 | """ 216 | test1: 217 | depends_components: 218 | - if: IDF_VERSION == "5.9.0" 219 | content: [ "VVV" ] 220 | - default: ["some_1", "some_2", "some_3"] 221 | - hello: 123 222 | 223 | """, 224 | encoding='utf8', 225 | ) 226 | try: 227 | with pytest.warns(UserWarning, match='Folder ".+" does not exist. Please check your manifest file'): 228 | Manifest.from_file(yaml_file) 229 | except InvalidManifest as e: 230 | assert str(e) == "Only the 'if' and 'default' keywords are supported in switch clause." 231 | 232 | yaml_file.write_text( 233 | """ 234 | test1: 235 | depends_components: 236 | - if: IDF_VERSION == "5.9.0" 237 | content: [ "VVV" ] 238 | - default: ["some_1", "some_2", "some_3"] 239 | - 123 240 | - 234 241 | """, 242 | encoding='utf8', 243 | ) 244 | try: 245 | Manifest.from_file(yaml_file) 246 | except InvalidManifest as e: 247 | assert str(e) == 'Current manifest format has to fit either the switch format or the list format.' 248 | 249 | def test_manifest_with_anchor(self, tmp_path, monkeypatch): 250 | yaml_file = tmp_path / 'test.yml' 251 | yaml_file.write_text( 252 | """ 253 | .base: &base 254 | depends_components: 255 | - a 256 | 257 | foo: &foo 258 | <<: *base 259 | disable: 260 | - if: IDF_TARGET == "esp32" 261 | 262 | bar: 263 | <<: *foo 264 | """, 265 | encoding='utf8', 266 | ) 267 | 268 | monkeypatch.setattr(idf_build_apps.manifest.manifest.FolderRule, 'DEFAULT_BUILD_TARGETS', ['esp32']) 269 | manifest = Manifest.from_file(yaml_file) 270 | assert manifest.enable_build_targets('bar') == [] 271 | 272 | def test_manifest_with_anchor_and_postfix(self, tmp_path): 273 | yaml_file = tmp_path / 'test.yml' 274 | 275 | yaml_file.write_text( 276 | """ 277 | foo: 278 | """, 279 | encoding='utf8', 280 | ) 281 | manifest = Manifest.from_file(yaml_file) 282 | assert manifest.enable_build_targets('foo') == sorted(SUPPORTED_TARGETS) 283 | 284 | yaml_file.write_text( 285 | """ 286 | .base_depends_components: &base-depends-components 287 | depends_components: 288 | - esp_hw_support 289 | - esp_rom 290 | - esp_wifi 291 | 292 | examples/wifi/coexist: 293 | <<: *base-depends-components 294 | depends_components+: 295 | - esp_coex 296 | depends_components-: 297 | - esp_rom 298 | """, 299 | encoding='utf8', 300 | ) 301 | 302 | manifest = Manifest.from_file(yaml_file) 303 | assert manifest.depends_components('examples/wifi/coexist') == ['esp_coex', 'esp_hw_support', 'esp_wifi'] 304 | 305 | yaml_file.write_text( 306 | """ 307 | .base: &base 308 | enable: 309 | - if: IDF_VERSION == "5.2.0" 310 | - if: IDF_VERSION == "5.3.0" 311 | 312 | foo: 313 | <<: *base 314 | enable+: 315 | - if: IDF_VERSION == "5.2.0" 316 | temp: true 317 | - if: IDF_VERSION == "5.4.0" 318 | reason: bar 319 | """, 320 | encoding='utf8', 321 | ) 322 | s_manifest_dict = parse(yaml_file) 323 | 324 | yaml_file.write_text( 325 | """ 326 | foo: 327 | enable: 328 | - if: IDF_VERSION == "5.3.0" 329 | - if: IDF_VERSION == "5.2.0" 330 | temp: true 331 | - if: IDF_VERSION == "5.4.0" 332 | reason: bar 333 | """, 334 | encoding='utf8', 335 | ) 336 | with open(yaml_file) as f: 337 | manifest_dict = yaml.safe_load(f) or {} 338 | 339 | assert s_manifest_dict['foo'] == manifest_dict['foo'] 340 | 341 | yaml_file.write_text( 342 | """ 343 | .base: &base 344 | enable: 345 | - if: IDF_VERSION == "5.3.0" 346 | 347 | foo: 348 | <<: *base 349 | enable+: 350 | - if: IDF_VERSION == "5.2.0" 351 | temp: true 352 | - if: IDF_VERSION == "5.4.0" 353 | reason: bar 354 | """, 355 | encoding='utf8', 356 | ) 357 | 358 | s_manifest_dict = parse(yaml_file) 359 | 360 | yaml_file.write_text( 361 | """ 362 | foo: 363 | enable: 364 | - if: IDF_VERSION == "5.3.0" 365 | - if: IDF_VERSION == "5.2.0" 366 | temp: true 367 | - if: IDF_VERSION == "5.4.0" 368 | reason: bar 369 | """, 370 | encoding='utf8', 371 | ) 372 | with open(yaml_file) as f: 373 | manifest_dict = yaml.safe_load(f) or {} 374 | 375 | assert s_manifest_dict['foo'] == manifest_dict['foo'] 376 | 377 | yaml_file.write_text( 378 | """ 379 | .test: &test 380 | depends_components: 381 | - a 382 | - b 383 | 384 | foo: 385 | <<: *test 386 | depends_components+: 387 | - c 388 | 389 | bar: 390 | <<: *test 391 | depends_components+: 392 | - d 393 | """, 394 | encoding='utf8', 395 | ) 396 | s_manifest_dict = parse(yaml_file) 397 | foo = s_manifest_dict['foo'] 398 | bar = s_manifest_dict['bar'] 399 | assert foo['depends_components'] == ['a', 'b', 'c'] 400 | assert bar['depends_components'] == ['a', 'b', 'd'] 401 | assert id(foo['depends_components']) != id(bar['depends_components']) 402 | 403 | yaml_file.write_text( 404 | """ 405 | .test: &test 406 | depends_components: 407 | - if: 1 408 | value: 123 409 | 410 | foo: 411 | <<: *test 412 | depends_components+: 413 | - if: 2 414 | value: 234 415 | 416 | bar: 417 | <<: *test 418 | depends_components+: 419 | - if: 2 420 | value: 345 421 | """, 422 | encoding='utf8', 423 | ) 424 | s_manifest_dict = parse(yaml_file) 425 | foo = s_manifest_dict['foo'] 426 | bar = s_manifest_dict['bar'] 427 | assert id(foo['depends_components']) != id(bar['depends_components']) 428 | print(s_manifest_dict) 429 | 430 | def test_manifest_postfix_order(self, tmp_path): 431 | yaml_file = tmp_path / 'test.yml' 432 | yaml_file.write_text( 433 | """ 434 | .base_depends_components: &base-depends-components 435 | depends_components: 436 | - esp_hw_support 437 | 438 | examples/wifi/coexist: 439 | <<: *base-depends-components 440 | depends_components+: 441 | - esp_coex 442 | depends_components-: 443 | - esp_coex 444 | """, 445 | encoding='utf8', 446 | ) 447 | 448 | manifest = Manifest.from_file(yaml_file) 449 | assert manifest.depends_components('examples/wifi/coexist') == ['esp_hw_support'] 450 | 451 | def test_from_files_duplicates(self, tmp_path, monkeypatch): 452 | yaml_file_1 = tmp_path / 'test1.yml' 453 | yaml_file_1.write_text( 454 | """ 455 | foo: 456 | enable: 457 | - if: IDF_TARGET == "esp32" 458 | """, 459 | encoding='utf8', 460 | ) 461 | 462 | yaml_file_2 = tmp_path / 'test2.yml' 463 | yaml_file_2.write_text( 464 | """ 465 | foo: 466 | enable: 467 | - if: IDF_TARGET == "esp32" 468 | """, 469 | encoding='utf8', 470 | ) 471 | 472 | monkeypatch.setattr(idf_build_apps.manifest.manifest.Manifest, 'CHECK_MANIFEST_RULES', True) 473 | folder_path = os.path.join(os.getcwd(), 'foo') 474 | os.makedirs(folder_path) 475 | 476 | with pytest.raises(InvalidManifest, match=f'Folder "{folder_path}" is already defined in {yaml_file_1!s}'): 477 | Manifest.from_files([str(yaml_file_1), str(yaml_file_2)]) 478 | 479 | monkeypatch.setattr(idf_build_apps.manifest.manifest.Manifest, 'CHECK_MANIFEST_RULES', False) 480 | Manifest.from_files([str(yaml_file_1), str(yaml_file_2)]) 481 | 482 | def test_manifest_dump_sha(self, tmp_path, sha_of_enable_only_esp32): 483 | yaml_file = tmp_path / 'test.yml' 484 | yaml_file.write_text( 485 | """ 486 | foo: 487 | enable: 488 | - if: IDF_TARGET == "esp32" 489 | bar: 490 | enable: 491 | - if: IDF_TARGET == "esp32" 492 | """, 493 | encoding='utf8', 494 | ) 495 | 496 | Manifest.from_file(yaml_file).dump_sha_values(str(tmp_path / '.sha')) 497 | 498 | with open('.sha') as f: 499 | assert f.readline() == f'bar:{sha_of_enable_only_esp32}\n' 500 | assert f.readline() == f'foo:{sha_of_enable_only_esp32}\n' 501 | 502 | def test_manifest_diff_sha(self, tmp_path, sha_of_enable_only_esp32): 503 | yaml_file = tmp_path / 'test.yml' 504 | yaml_file.write_text( 505 | """ 506 | foo: 507 | enable: 508 | - if: IDF_TARGET == "esp32" 509 | - if: IDF_TARGET == "esp32c3" 510 | bar: 511 | enable: 512 | - if: IDF_TARGET == "esp32" 513 | baz: 514 | enable: 515 | - if: IDF_TARGET == "esp32" 516 | """, 517 | encoding='utf8', 518 | ) 519 | 520 | with open('.sha', 'w') as fw: 521 | fw.write(f'bar:{sha_of_enable_only_esp32}\n') 522 | fw.write('\n') # test empty line 523 | fw.write(' ') # test spaces 524 | fw.write(f'foo:{sha_of_enable_only_esp32}\n') 525 | 526 | assert Manifest.from_file(yaml_file).diff_sha_with_filepath(str(tmp_path / '.sha')) == { 527 | 'baz', 528 | 'foo', 529 | } 530 | 531 | def test_folder_rule_introduced_by(self, tmp_path): 532 | yaml_file = tmp_path / 'test.yml' 533 | yaml_file.write_text( 534 | """ 535 | foo: 536 | enable: 537 | - if: IDF_TARGET == "esp32" 538 | - if: IDF_TARGET == "esp32c3" 539 | bar: 540 | enable: 541 | - if: IDF_TARGET == "esp32" 542 | baz: 543 | enable: 544 | - if: IDF_TARGET == "esp32" 545 | """, 546 | encoding='utf8', 547 | ) 548 | 549 | manifest = Manifest.from_file(yaml_file) 550 | assert manifest.most_suitable_rule('baz').by_manifest_file == str(yaml_file) 551 | 552 | 553 | class TestIfParser: 554 | def test_invalid_if_statement(self): 555 | statement = '1' 556 | with pytest.raises(InvalidIfClause, match='Invalid if clause: 1'): 557 | IfClause(statement) 558 | 559 | def test_temporary_must_with_reason(self): 560 | with pytest.raises(InvalidIfClause, match='"reason" must be set when "temporary: true"'): 561 | IfClause(stmt='IDF_TARGET == "esp32"', temporary=True) 562 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD 2 | # SPDX-License-Identifier: Apache-2.0 3 | import os 4 | from pathlib import ( 5 | Path, 6 | ) 7 | 8 | import pytest 9 | 10 | from idf_build_apps.utils import ( 11 | files_matches_patterns, 12 | get_parallel_start_stop, 13 | rmdir, 14 | to_absolute_path, 15 | ) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | 'patterns, expected', 20 | [ 21 | ('*.txt', ['test/inner', 'test/inner/test.txt', 'test/test.txt']), 22 | ( 23 | ['*.txt', '*.log'], 24 | [ 25 | 'test/inner', 26 | 'test/inner/build.log', 27 | 'test/inner/test.txt', 28 | 'test/test.txt', 29 | ], 30 | ), 31 | ], 32 | ) 33 | def test_rmdir(tmp_path, patterns, expected): 34 | test_dir = tmp_path / 'test' 35 | test_dir.mkdir() 36 | dir1 = test_dir / 'inner' 37 | dir1.mkdir() 38 | (test_dir / 'inner2').mkdir() 39 | 40 | Path(dir1 / 'test.txt').touch() 41 | Path(dir1 / 'build.log').touch() 42 | Path(test_dir / 'test.txt').touch() 43 | 44 | rmdir(test_dir, exclude_file_patterns=patterns) 45 | 46 | assert sorted(Path(test_dir).glob('**/*')) == [Path(tmp_path / i) for i in expected] 47 | 48 | 49 | @pytest.mark.parametrize( 50 | 'total, parallel_count, parallel_index, start, stop', 51 | [ 52 | (1, 1, 1, 1, 1), 53 | (1, 2, 2, 2, 1), 54 | (1, 10, 1, 1, 1), 55 | (6, 4, 1, 1, 2), 56 | (6, 4, 2, 3, 4), 57 | (6, 4, 3, 5, 6), 58 | (6, 4, 4, 7, 6), 59 | (10, 10, 9, 9, 9), 60 | (33, 2, 1, 1, 17), 61 | (33, 2, 2, 18, 33), 62 | ], 63 | ) 64 | def test_get_parallel_start_stop(total, parallel_count, parallel_index, start, stop): 65 | assert (start, stop) == get_parallel_start_stop(total, parallel_count, parallel_index) 66 | 67 | 68 | def test_files_matches_patterns(tmp_path): 69 | # used for testing absolute paths 70 | temp_dir = tmp_path / 'temp' 71 | temp_dir.mkdir() 72 | os.chdir(temp_dir) 73 | 74 | # create real files 75 | test_dir = tmp_path / 'test' 76 | test_dir.mkdir() 77 | 78 | a_dir = test_dir / 'a' 79 | a_dir.mkdir() 80 | b_dir = a_dir / 'b' 81 | b_dir.mkdir() 82 | c_dir = b_dir / 'c' 83 | c_dir.mkdir() 84 | b_py = Path(a_dir / 'b.py') 85 | c_py = Path(b_dir / 'c.py') 86 | d_py = Path(c_dir / 'd.py') 87 | b_py.touch() 88 | c_py.touch() 89 | d_py.touch() 90 | 91 | # ├── temp 92 | # └── test 93 | # └── a 94 | # ├── b 95 | # │ ├── c 96 | # │ │ └── d.py 97 | # │ └── c.py 98 | # ├── .hidden 99 | # └── b.py 100 | # 101 | 102 | # in correct relative path 103 | for matched_files, pat, rootpath in [ 104 | ([b_py], '*.py', a_dir), 105 | ([b_py, c_py, d_py], '**/*.py', a_dir), 106 | ([c_py], '*.py', b_dir), 107 | ([c_py, d_py], '**/*.py', b_dir), 108 | ([d_py], '*.py', c_dir), 109 | ([d_py], '**/*.py', c_dir), 110 | ]: 111 | for f in matched_files: 112 | assert files_matches_patterns(f, pat, rootpath) 113 | 114 | # in None root path with relative pattern 115 | for matched_files, pat in [ 116 | ([b_py], 'a/*.py'), 117 | ([b_py, c_py, d_py], 'a/**/*.py'), 118 | ([c_py], 'a/b/*.py'), 119 | ([c_py, d_py], 'a/b/**/*.py'), 120 | ([d_py], 'a/b/c/*.py'), 121 | ([d_py], 'a/b/c/**/*.py'), 122 | ]: 123 | for f in matched_files: 124 | # with correct pwd 125 | os.chdir(test_dir) 126 | assert files_matches_patterns(f, pat) 127 | 128 | # with wrong pwd 129 | os.chdir(temp_dir) 130 | assert not files_matches_patterns(f, pat) 131 | 132 | # use correct absolute path, in wrong pwd 133 | for matched_files, pat in [ 134 | ([b_py], 'a/*.py'), 135 | ([b_py, c_py, d_py], 'a/**/*.py'), 136 | ([c_py], 'a/b/*.py'), 137 | ([c_py, d_py], 'a/b/**/*.py'), 138 | ([d_py], 'a/b/c/*.py'), 139 | ([d_py], 'a/b/c/**/*.py'), 140 | ]: 141 | abs_pat = to_absolute_path(pat, test_dir) 142 | os.chdir(temp_dir) 143 | for f in matched_files: 144 | assert files_matches_patterns(f, abs_pat) 145 | --------------------------------------------------------------------------------