├── .python-version ├── tests ├── test_otaclient │ ├── __init__.py │ ├── test_ota_core │ │ ├── __init__.py │ │ └── conftest.py │ ├── test_create_standby │ │ ├── __init__.py │ │ └── test_resume_ota.py │ ├── test_grpc │ │ ├── __init__.py │ │ └── test_api_v2 │ │ │ └── __init__.py │ ├── test_boot_control │ │ ├── __init__.py │ │ ├── fstab_origin │ │ ├── fstab_updated │ │ ├── default_grub │ │ ├── extlinux.conf_slot_a │ │ └── extlinux.conf_slot_b │ ├── test_configs │ │ ├── test_cfg_consts.py │ │ └── test_cfg_configurable.py │ ├── test_log_setting.py │ └── test_utils.py ├── test_ota_metadata │ ├── __init__.py │ ├── test_legacy2 │ │ ├── __init__.py │ │ └── test_e2e.py │ ├── test_ota_image_v1 │ │ ├── __ini__.py │ │ ├── test_cert_verification.py │ │ └── test_e2e.py │ ├── conftest.py │ ├── test_detect_ota_image_spec.py │ └── test_ca_store.py ├── test_otaclient_api │ ├── __init__.py │ └── test_v2 │ │ └── __init__.py ├── test_otaclient_common │ ├── __init__.py │ ├── test_proto_wrapper │ │ ├── __init__.py │ │ ├── example.proto │ │ ├── example_pb2_wrapper.py │ │ ├── example_pb2.py │ │ └── example_pb2.pyi │ ├── test_typing.py │ └── test_logging.py ├── data │ ├── ota_image_builder │ │ ├── sys_config.yaml │ │ └── full_annotations.yaml │ ├── client_package │ │ ├── v1.squashfs │ │ ├── v1_v2.patch │ │ └── README.md │ ├── extlinux.conf-r35.4.1-template2 │ ├── extlinux.conf-r35.4.1-updated2 │ ├── extlinux.conf-r35.4.1-template1 │ └── extlinux.conf-r35.4.1-updated1 ├── __init__.py ├── test_ota_proxy │ ├── __init__.py │ ├── test_subprocess_launch_otaproxy.py │ ├── test_cache_control_header.py │ └── test_utils.py ├── custom.cfg └── keys │ └── gen_certs.sh ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE │ ├── fix.md │ ├── refinement.md │ └── feature.md ├── dependabot.yml ├── actions │ ├── calculate_checksum │ │ └── action.yml │ ├── generate_manifest │ │ └── action.yml │ ├── build_squashfs_image │ │ └── action.yml │ └── build_patches │ │ └── action.yml ├── renovate.json5 ├── release.yml └── workflows │ ├── issue-metrics.yml │ ├── build_test_base_images.yaml │ ├── release_api.yml │ ├── test.yaml │ └── sync_lockfile.yaml ├── tools ├── .gitignore ├── offline_ota_image_builder │ └── README.md ├── local_ota │ ├── README.md │ ├── __init__.py │ └── api_stop_v2.py └── status_monitor │ ├── configs.py │ └── __main__.py ├── proto ├── .gitignore ├── src │ └── otaclient_pb2 │ │ ├── .gitignore │ │ ├── v2 │ │ ├── .gitignore │ │ └── __init__.py │ │ ├── README.md │ │ └── __init__.py ├── whl │ ├── otaclient_pb2-0.5.0.33632fb-py3-none-any.whl │ └── otaclient_pb2-0.6.0.80554b2-py3-none-any.whl ├── README.md ├── ota_metafiles.proto ├── pyproject.toml └── hatch_build.py ├── .markdownlintignore ├── src ├── .gitignore ├── ota_metadata │ ├── README.md │ ├── utils │ │ ├── __init__.py │ │ └── detect_ota_image_ver.py │ ├── legacy2 │ │ ├── __init__.py │ │ └── rs_table.py │ ├── errors.py │ └── file_table │ │ └── __init__.py ├── otaclient │ ├── grpc │ │ ├── README.md │ │ ├── __init__.py │ │ └── api_v2 │ │ │ ├── __init__.py │ │ │ └── main.py │ ├── __main__.py │ ├── __init__.py │ ├── boot_control │ │ ├── __init__.py │ │ ├── protocol.py │ │ └── configs.py │ ├── ota_core │ │ ├── README.md │ │ ├── __init__.py │ │ └── _download_bsp_version_file.py │ ├── create_standby │ │ ├── _common.py │ │ ├── __init__.py │ │ └── utils.py │ └── configs │ │ ├── _common.py │ │ ├── __init__.py │ │ └── cfg.py ├── otaclient_api │ └── v2 │ │ ├── README.md │ │ ├── __init__.py │ │ └── api_stub.py ├── ota_proxy │ ├── errors.py │ ├── _consts.py │ ├── utils.py │ ├── external_cache.py │ ├── __init__.py │ └── __main__.py ├── otaclient_manifest │ ├── __init__.py │ └── schema.py └── otaclient_common │ ├── download_info.py │ ├── _env.py │ └── __init__.py ├── .markdownlint.yaml ├── .cspell.json ├── samples ├── ecu_info.yaml ├── otaclient.service ├── README.md ├── proxy_info.yaml └── otaclient.service_app ├── sonar-project.properties ├── docker ├── test_base │ ├── docker-compose_tests_py313.yml │ ├── docker-compose_build.yml │ ├── docker-compose_tests.yml │ ├── entry_point.sh │ ├── Dockerfile │ ├── entry_point_py313.sh │ └── README.md ├── mini_ota_img │ ├── README.md │ ├── sys_img.Dockerfile │ ├── new_ota_image.Dockerfile │ └── ota_image.Dockerfile └── app_img │ ├── README.md │ └── Dockerfile ├── docs └── update_phase_transition.md ├── LICENSE.md ├── .pre-commit-config.yaml ├── README.md ├── .devcontainer └── devcontainer.json └── scripts └── hatch_build_lock_deps.py /.python-version: -------------------------------------------------------------------------------- 1 | 3.8 2 | -------------------------------------------------------------------------------- /tests/test_otaclient/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_otaclient_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @tier4/ota 2 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | update_request.yaml 2 | -------------------------------------------------------------------------------- /proto/.gitignore: -------------------------------------------------------------------------------- 1 | .venv/ 2 | dist/ 3 | build/ 4 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_legacy2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_ota_core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_otaclient_api/test_v2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_ota_image_v1/__ini__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_create_standby/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.markdownlintignore: -------------------------------------------------------------------------------- 1 | LICENSE.md 2 | docs/SERVICES.md 3 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | # generated version file by build 2 | _otaclient_version.py 3 | -------------------------------------------------------------------------------- /proto/src/otaclient_pb2/.gitignore: -------------------------------------------------------------------------------- 1 | # generated version file by build 2 | _version.py 3 | -------------------------------------------------------------------------------- /proto/src/otaclient_pb2/v2/.gitignore: -------------------------------------------------------------------------------- 1 | !__init__.py 2 | *.py 3 | *.pyi 4 | *.proto 5 | -------------------------------------------------------------------------------- /src/ota_metadata/README.md: -------------------------------------------------------------------------------- 1 | # OTA image metadata 2 | 3 | Libs for parsing OTA image. 4 | -------------------------------------------------------------------------------- /proto/src/otaclient_pb2/README.md: -------------------------------------------------------------------------------- 1 | # otaclient API version2 generated protobuf code 2 | 3 | DO NOT EDIT! 4 | -------------------------------------------------------------------------------- /tests/data/ota_image_builder/sys_config.yaml: -------------------------------------------------------------------------------- 1 | hostname: autoware 2 | persist_files: 3 | - "/etc/netplan" 4 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | "MD013": false 2 | "MD041": false 3 | "MD024": 4 | "siblings_only": true 5 | "MD029": false 6 | -------------------------------------------------------------------------------- /src/otaclient/grpc/README.md: -------------------------------------------------------------------------------- 1 | # OTAClient gRPC Interface 2 | 3 | The implementation of the OTAClient gRPC interface. 4 | -------------------------------------------------------------------------------- /tests/data/client_package/v1.squashfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tier4/ota-client/HEAD/tests/data/client_package/v1.squashfs -------------------------------------------------------------------------------- /tests/data/client_package/v1_v2.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tier4/ota-client/HEAD/tests/data/client_package/v1_v2.patch -------------------------------------------------------------------------------- /proto/whl/otaclient_pb2-0.5.0.33632fb-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tier4/ota-client/HEAD/proto/whl/otaclient_pb2-0.5.0.33632fb-py3-none-any.whl -------------------------------------------------------------------------------- /proto/whl/otaclient_pb2-0.6.0.80554b2-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tier4/ota-client/HEAD/proto/whl/otaclient_pb2-0.6.0.80554b2-py3-none-any.whl -------------------------------------------------------------------------------- /src/otaclient_api/v2/README.md: -------------------------------------------------------------------------------- 1 | # OTAClient API version 2 2 | 3 | Package for holding the protobuf python pb2 generated files and related wrappers and libs for OTAClient API version 2. 4 | -------------------------------------------------------------------------------- /tools/offline_ota_image_builder/README.md: -------------------------------------------------------------------------------- 1 | # Offline OTA image builder 2 | 3 | Please use this version instead: . 4 | -------------------------------------------------------------------------------- /.cspell.json: -------------------------------------------------------------------------------- 1 | { 2 | "words": [ 3 | "otaclient", 4 | "ota_proxy", 5 | "otaproxy", 6 | "zstandard", 7 | "zst", 8 | "zstd" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Iterable 2 | 3 | 4 | def iter_helper(_iter: Iterable[Any]) -> int: 5 | _count = 0 6 | for _count, _ in enumerate(_iter, start=1): 7 | ... 8 | return _count 9 | -------------------------------------------------------------------------------- /samples/ecu_info.yaml: -------------------------------------------------------------------------------- 1 | # This is the sample ecu_info.yaml for a single x86_64 ECU setup. 2 | # Please check ecu_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/AGmpqFFc. 3 | format_version: 1 4 | ecu_id: autoware 5 | bootloader: grub 6 | available_ecu_ids: 7 | - autoware 8 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.organization=tier4 2 | sonar.projectKey=tier4_ota-client 3 | sonar.python.coverage.reportPaths=test_result/coverage.xml 4 | sonar.sources=./src 5 | sonar.tests=tests 6 | sonar.sourceEncoding=UTF-8 7 | sonar.python.version=3.8,3.9,3.10,3.11,3.12,3.13 8 | sonar.exclusions=**/*_pb2* 9 | -------------------------------------------------------------------------------- /samples/otaclient.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OTA Client 3 | After=network-online.target nss-lookup.target 4 | Wants=network-online.target 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart=/opt/ota/client/venv/bin/python3 -m otaclient 9 | Restart=always 10 | RestartSec=16 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /samples/README.md: -------------------------------------------------------------------------------- 1 | # OTAClient configuration files samples 2 | 3 | This folder contains the sample otaclient configuration files **ecu_info.yaml**, **proxy_info.yaml** and systemd service unit file **otaclient.service**(for running otaclient with python venv) or **otaclient.service_app**(for running otaclient as systemd managed app image) for a single ECU OTA setup. 4 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | This is the main entry for selecting a template for your PR. 2 | 3 | **Click** the `Preview` tab and **select** a PR template according to PR type: 4 | 5 | - [Feature](?expand=1&template=feature.md) 6 | - [Fix](?expand=1&template=fix.md) 7 | - [Refinement](?expand=1&template=refinement.md) 8 | 9 | If PR type is not in the above list, feel free to start from blank PR body. 10 | 11 | **DON'T INCLUDE THIS PAGE'S CONTENTS IN YOUR PR BODY.** 12 | -------------------------------------------------------------------------------- /docker/test_base/docker-compose_tests_py313.yml: -------------------------------------------------------------------------------- 1 | x-common: &_common 2 | network_mode: bridge 3 | environment: 4 | OUTPUT_DIR: /test_result 5 | CERTS_DIR: /certs 6 | volumes: 7 | - ../..:/otaclient_src:ro 8 | - ../../test_result:/test_result:rw 9 | - ./entry_point_py313.sh:/entry_point.sh:ro 10 | 11 | services: 12 | tester-ubuntu-22.04: 13 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_22.04 14 | container_name: ota-test_ubuntu2204 15 | <<: *_common 16 | -------------------------------------------------------------------------------- /samples/proxy_info.yaml: -------------------------------------------------------------------------------- 1 | # This is the sample proxy_info.yaml for a single ECU setup. 2 | # Please check proxy_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/qT4N4K0X. 3 | format_version: 1 4 | enable_local_ota_proxy: true 5 | enable_local_ota_proxy_cache: true 6 | local_ota_proxy_listen_addr: 127.0.0.1 7 | local_ota_proxy_listen_port: 8082 8 | # if otaclient-logger is installed locally 9 | logging_server: "http://127.0.0.1:8083" 10 | logging_server_grpc: "http://127.0.0.1:8084" 11 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/fix.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | 5 | ## Check list 6 | 7 | 8 | 9 | - [ ] test files that cover the bug case(s) are implemented. 10 | - [ ] local tests are passing. 11 | 12 | ## Bug fix 13 | 14 | ### Current behavior 15 | 16 | ### Behavior after fix 17 | 18 | ## Related links & ticket 19 | 20 | 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /tools/local_ota/README.md: -------------------------------------------------------------------------------- 1 | # local OTA 2 | 3 | Tools for triggering OTA locally. 4 | 5 | Currently supports API version 2. 6 | 7 | ## Usage 8 | 9 | 1. define the `update_request.yaml` as follow: 10 | 11 | ```yaml 12 | # adjust the settings as your needs 13 | - ecu_id: "autoware" 14 | version: "789.x" 15 | url: "https://10.0.1.1:8443/" 16 | cookies: '{"test": "my-cookie"}' 17 | ``` 18 | 19 | 1. make the OTA update request API call as follow: 20 | 21 | ```shell 22 | python3 tools/local_ota/api_v2.py -i 10.0.1.10 update_request.yaml 23 | ``` 24 | -------------------------------------------------------------------------------- /docker/mini_ota_img/README.md: -------------------------------------------------------------------------------- 1 | # Mini OTA image for deterministic OTAClient test 2 | 3 | Repo: [ota_img_for_test](https://github.com/tier4/ota-client/pkgs/container/ota-client%2Fota_img_for_test) 4 | 5 | Old OTA image: `ghcr.io/tier4/ota-client/ota_img_for_test:ubuntu_22.04` 6 | New OTA image: `ghcr.io/tier4/ota-client/ota_img_for_test:ubuntu_22.04-ota_image_v1` 7 | 8 | ## Build OTA image 9 | 10 | ```bash 11 | # at the root of this repo 12 | sudo docker build \ 13 | -f docker/mini_ota_img/new_ota_image.Dockerfile -t ghcr.io/tier4/ota-client/ota_img_for_test:ubuntu_22.04-ota_image_v1 . 14 | ``` 15 | -------------------------------------------------------------------------------- /src/ota_proxy/errors.py: -------------------------------------------------------------------------------- 1 | class BaseOTACacheError(Exception): ... 2 | 3 | 4 | class CacheMultiStreamingFailed(BaseOTACacheError): ... 5 | 6 | 7 | class CacheStreamingFailed(BaseOTACacheError): ... 8 | 9 | 10 | class StorageReachHardLimit(BaseOTACacheError): ... 11 | 12 | 13 | class CacheStreamingInterrupt(BaseOTACacheError): ... 14 | 15 | 16 | class CacheCommitFailed(BaseOTACacheError): ... 17 | 18 | 19 | class ReaderPoolBusy(Exception): 20 | """Raised when read worker thread pool is busy.""" 21 | 22 | 23 | class CacheProviderNotReady(Exception): 24 | """Raised when subscriber timeout waiting cache provider.""" 25 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tools/local_ota/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/ota_metadata/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/otaclient/grpc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/otaclient_manifest/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/test_ota_proxy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/otaclient/grpc/api_v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_grpc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_grpc/test_api_v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_proto_wrapper/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /proto/README.md: -------------------------------------------------------------------------------- 1 | # OTA Service API proto 2 | 3 | This folder includes the OTA service API proto file, and is also a configured python package. 4 | You can use `hatch` to simply build the OTA service API python package for building grpc server or client. 5 | 6 | ## How to build 7 | 8 | 1. setup a venv and install hatch: 9 | 10 | ```shell 11 | python3 -m venv .venv 12 | # enable the venv 13 | . .venv/bin/activate 14 | python3 -m pip install -U pip 15 | python3 -m pip install hatch 16 | ``` 17 | 18 | 2. build the wheel package with hatch 19 | 20 | ```shell 21 | # enable the venv 22 | . .venv/bin/activate 23 | hatch build -t wheel 24 | ``` 25 | 26 | 3. the built package will be placed under `./dist` folder 27 | -------------------------------------------------------------------------------- /src/otaclient_api/v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """OTAClient API, version 2.""" 15 | -------------------------------------------------------------------------------- /docs/update_phase_transition.md: -------------------------------------------------------------------------------- 1 | ```mermaid 2 | stateDiagram-v2 3 | [*] --> INITIALIZING : wake up / session started / session finished 4 | INITIALIZING --> PROCESSING_METADATA : metadata processing is started 5 | PROCESSING_METADATA --> CALCULATING_DELTA : delta calculation is started 6 | CALCULATING_DELTA --> DOWNLOADING_OTA_FILES : delta resource downloading is started 7 | DOWNLOADING_OTA_FILES --> APPLYING_UPDATE : update applying is started 8 | APPLYING_UPDATE --> PROCESSING_POSTUPDATE : post update processing is started 9 | PROCESSING_POSTUPDATE --> FINALIZING_UPDATE : finalizing update is started 10 | PROCESSING_METADATA --> DOWNLOADING_OTA_CLIENT : client downloading is started 11 | -------------------------------------------------------------------------------- /proto/ota_metafiles.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package otaclient; 4 | 5 | message RegularInf { 6 | int32 mode = 1; 7 | int32 uid = 2; 8 | int32 gid = 3; 9 | int32 nlink = 4; 10 | bytes sha256hash = 5; 11 | string path = 6; 12 | int64 size = 7; 13 | int64 inode = 8; 14 | string compressed_alg = 9; 15 | } 16 | 17 | message DirectoryInf { 18 | int32 mode = 1; 19 | int32 uid = 2; 20 | int32 gid = 3; 21 | string path = 4; 22 | } 23 | 24 | message SymbolicLinkInf { 25 | int32 mode = 1; 26 | int32 uid = 2; 27 | int32 gid = 3; 28 | string slink = 4; 29 | string srcpath = 5; 30 | } 31 | 32 | message PersistentInf { 33 | string path = 1; 34 | } 35 | -------------------------------------------------------------------------------- /docker/test_base/docker-compose_build.yml: -------------------------------------------------------------------------------- 1 | # compose file for building the test base images 2 | 3 | services: 4 | tester-ubuntu-20.04: 5 | build: 6 | context: . 7 | args: 8 | UBUNTU_BASE: ubuntu:20.04 9 | pull_policy: never 10 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_20.04 11 | 12 | tester-ubuntu-22.04: 13 | build: 14 | context: . 15 | args: 16 | UBUNTU_BASE: ubuntu:22.04 17 | pull_policy: never 18 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_22.04 19 | 20 | tester-ubuntu-24.04: 21 | build: 22 | context: . 23 | args: 24 | UBUNTU_BASE: ubuntu:24.04 25 | pull_policy: never 26 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_24.04 27 | -------------------------------------------------------------------------------- /docker/test_base/docker-compose_tests.yml: -------------------------------------------------------------------------------- 1 | x-common: &_common 2 | network_mode: bridge 3 | environment: 4 | OUTPUT_DIR: /test_result 5 | CERTS_DIR: /certs 6 | volumes: 7 | - ../..:/otaclient_src:ro 8 | - ../../test_result:/test_result:rw 9 | - ./entry_point.sh:/entry_point.sh:ro 10 | 11 | services: 12 | tester-ubuntu-20.04: 13 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_20.04 14 | container_name: ota-test_ubuntu2004 15 | <<: *_common 16 | 17 | tester-ubuntu-22.04: 18 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_22.04 19 | container_name: ota-test_ubuntu2204 20 | <<: *_common 21 | 22 | tester-ubuntu-24.04: 23 | image: ghcr.io/tier4/ota-client/test_base:ubuntu_24.04 24 | container_name: ota-test_ubuntu2404 25 | <<: *_common 26 | -------------------------------------------------------------------------------- /src/ota_metadata/legacy2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """OTA image metadata, legacy version.""" 15 | 16 | SUPORTED_COMPRESSION_TYPES = ("zst", "zstd") 17 | DIGEST_ALG = "sha256" 18 | -------------------------------------------------------------------------------- /proto/src/otaclient_pb2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from otaclient_pb2._version import version, version_tuple 17 | 18 | __version__ = version 19 | __version_tuple__ = version_tuple 20 | -------------------------------------------------------------------------------- /src/otaclient/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from multiprocessing import freeze_support 16 | 17 | if __name__ == "__main__": 18 | freeze_support() 19 | 20 | from otaclient import main 21 | 22 | main.main() 23 | -------------------------------------------------------------------------------- /proto/src/otaclient_pb2/v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = version = "2.2.0" 16 | __version_tuple__ = version_tuple = (2, 2, 0) 17 | 18 | __all__ = ["version", "__version__", "version_tuple", "__version_tuple__"] 19 | -------------------------------------------------------------------------------- /src/otaclient/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | try: 16 | from _otaclient_version import __version__, version 17 | except ImportError: 18 | # unknown version 19 | version = __version__ = "0.0.0" 20 | 21 | __all__ = ["version", "__version__"] 22 | -------------------------------------------------------------------------------- /tests/data/client_package/README.md: -------------------------------------------------------------------------------- 1 | # How to create test squashfs and patch files 2 | 3 | This guide demonstrates how to create test squashfs files and generate a test patch between versions. 4 | 5 | ## Steps 6 | 7 | 1. Create two sample directories: 8 | 9 | ```bash 10 | mkdir dir_v1 dir_v2 11 | ``` 12 | 13 | 2. Add test files with different content: 14 | 15 | ```bash 16 | echo "Hello v1" > dir_v1/test.txt 17 | echo "Hello v2" > dir_v2/test.txt 18 | ``` 19 | 20 | 3. Create squashfs archives for both versions: 21 | 22 | ```bash 23 | mksquashfs dir_v1 v1.squashfs -comp gzip 24 | mksquashfs dir_v2 v2.squashfs -comp gzip 25 | ``` 26 | 27 | 4. Generate a patch from v1 to v2 using zstd: 28 | 29 | ```bash 30 | zstd --patch-from=v1.squashfs v2.squashfs -o v1_v2.patch 31 | ``` 32 | 33 | 5. Clean up: 34 | 35 | ```bash 36 | rm -rf v2.squashfs 37 | rm -rf dir_v1 dir_v2 38 | ``` 39 | -------------------------------------------------------------------------------- /samples/otaclient.service_app: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OTAClient 3 | After=network-online.target nss-lookup.target 4 | Wants=network-online.target 5 | 6 | [Service] 7 | Type=simple 8 | Environment=RUNNING_AS_APP_IMAGE=yes 9 | RootImage=/opt/ota/client/otaclient.squashfs 10 | ExecStart=/otaclient/otaclient 11 | 12 | BindPaths=/boot:/boot:rbind 13 | BindPaths=/dev:/dev 14 | BindPaths=/dev/shm:/dev/shm 15 | BindPaths=/etc:/etc 16 | BindPaths=/opt:/opt 17 | BindPaths=/proc:/proc 18 | BindPaths=/root:/root 19 | BindPaths=/sys:/sys:rbind 20 | BindPaths=/run:/run 21 | BindReadOnlyPaths=-/usr/share/ca-certificates:/usr/share/ca-certificates 22 | BindReadOnlyPaths=-/usr/local/share/ca-certificates:/usr/local/share/ca-certificates 23 | BindReadOnlyPaths=-/usr/share/zoneinfo:/usr/share/zoneinfo 24 | BindPaths=/:/host_root:rbind 25 | 26 | Restart=always 27 | RestartSec=16 28 | 29 | [Install] 30 | WantedBy=multi-user.target 31 | -------------------------------------------------------------------------------- /src/ota_metadata/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Common errors that might be raised during parsing OTA image metadata.""" 15 | 16 | 17 | class SignCertInvalid(Exception): ... 18 | 19 | 20 | class ImageMetadataInvalid(Exception): ... 21 | 22 | 23 | class NoCAStoreAvailable(Exception): ... 24 | -------------------------------------------------------------------------------- /src/ota_proxy/_consts.py: -------------------------------------------------------------------------------- 1 | from multidict import istr 2 | 3 | from .cache_control_header import OTAFileCacheControl 4 | 5 | # uvicorn 6 | REQ_TYPE_LIFESPAN = "lifespan" 7 | REQ_TYPE_HTTP = "http" 8 | RESP_TYPE_BODY = "http.response.body" 9 | RESP_TYPE_START = "http.response.start" 10 | 11 | METHOD_GET = "GET" 12 | 13 | # headers 14 | # for implementation convienience, we use lowercase for all headers. 15 | HEADER_OTA_FILE_CACHE_CONTROL = istr(OTAFileCacheControl.HEADER_LOWERCASE) 16 | HEADER_AUTHORIZATION = istr("authorization") 17 | HEADER_COOKIE = istr("cookie") 18 | HEADER_CONTENT_ENCODING = istr("content-encoding") 19 | HEADER_CONTENT_TYPE = istr("content-type") 20 | BHEADER_OTA_FILE_CACHE_CONTROL = OTAFileCacheControl.HEADER_LOWERCASE.encode("utf-8") 21 | BHEADER_AUTHORIZATION = b"authorization" 22 | BHEADER_COOKIE = b"cookie" 23 | BHEADER_CONTENT_ENCODING = b"content-encoding" 24 | BHEADER_CONTENT_TYPE = b"content-type" 25 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/fstab_origin: -------------------------------------------------------------------------------- 1 | # /etc/fstab: static file system information. 2 | # 3 | # Use 'blkid' to print the universally unique identifier for a 4 | # device; this may be used with UUID= as a more robust way to name devices 5 | # that works even if disks are added and removed. See fstab(5). 6 | # 7 | # 8 | # / was on /dev/sda2 during installation 9 | UUID=44d14df0-31be-4fe2-b05a-bae6cf157ea4 / ext4 errors=remount-ro 0 1 10 | # /boot was on /dev/sda1 during installation 11 | UUID=ca573144-b0a4-44f6-a750-45a169c3c6e7 /boot ext4 defaults 0 2 12 | /swapfile none swap sw 0 0 13 | tmpfs /media/autoware/LOG tmpfs rw,nosuid,nodev,noexec,nofail,size=10G,mode=1755 0 0 14 | LABEL=ROSBAG /media/autoware/ROSBAG ext4 nofail 0 0 15 | tmpfs /mnt/LOG tmpfs rw,nosuid,nodev,noexec,nofail,size=10G,mode=1755 0 0 16 | LABEL=ROSBAG /mnt/ROSBAG ext4 nofail 0 0 17 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/fstab_updated: -------------------------------------------------------------------------------- 1 | # /etc/fstab: static file system information. 2 | # 3 | # Use 'blkid' to print the universally unique identifier for a 4 | # device; this may be used with UUID= as a more robust way to name devices 5 | # that works even if disks are added and removed. See fstab(5). 6 | # 7 | # 8 | # / was on /dev/sda2 during installation 9 | UUID=bbbbbbbb-1111-1111-1111-bbbbbbbbbbbb / ext4 errors=remount-ro 0 1 10 | # /boot was on /dev/sda1 during installation 11 | UUID=ca573144-b0a4-44f6-a750-45a169c3c6e7 /boot ext4 defaults 0 2 12 | /swapfile none swap sw 0 0 13 | tmpfs /media/autoware/LOG tmpfs rw,nosuid,nodev,noexec,nofail,size=10G,mode=1755 0 0 14 | LABEL=ROSBAG /media/autoware/ROSBAG ext4 nofail 0 0 15 | tmpfs /mnt/LOG tmpfs rw,nosuid,nodev,noexec,nofail,size=10G,mode=1755 0 0 16 | LABEL=ROSBAG /mnt/ROSBAG ext4 nofail 0 0 17 | -------------------------------------------------------------------------------- /.github/actions/calculate_checksum/action.yml: -------------------------------------------------------------------------------- 1 | name: "Calculate Checksum" 2 | description: "Calculates SHA256 checksums for specified file patterns" 3 | inputs: 4 | file_patterns: 5 | description: "File patterns to calculate checksums in the specified directory. Only single glob pattern is supported." 6 | required: true 7 | default: "*.{whl,squashfs,patch,json}" 8 | directory: 9 | description: "Directory containing the files. Only single directory is supported." 10 | required: true 11 | default: "dist" 12 | 13 | runs: 14 | using: "composite" 15 | steps: 16 | - name: Calculate checksum 17 | shell: bash 18 | run: | 19 | for FILE in ${{ inputs.directory }}/${{ inputs.file_patterns }}; do 20 | if [ -s "${FILE}" ]; then 21 | sha256sum "${FILE}" | awk '{print "sha256:"$1}' > "${FILE}.checksum" 22 | echo "Generated checksum for \"${FILE}\"" 23 | fi 24 | done 25 | -------------------------------------------------------------------------------- /tests/custom.cfg: -------------------------------------------------------------------------------- 1 | menuentry 'Ubuntu, with Linux 5.4.0-73-generic' --class ubuntu --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-5.4.0-73-generic-advanced-01234567-0123-0123-0123-0123456789ab' { 2 | recordfail 3 | load_video 4 | gfxmode $linux_gfx_mode 5 | insmod gzio 6 | if [ x$grub_platform = xxen ]; then insmod xzio; insmod lzopio; fi 7 | insmod part_gpt 8 | insmod ext2 9 | set root='hd0,gpt2' 10 | if [ x$feature_platform_search_hint = xy ]; then 11 | search --no-floppy --fs-uuid --set=root --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2 ad35fc7d-d90f-4a98-84ae-fd65aff1f535 12 | else 13 | search --no-floppy --fs-uuid --set=root ad35fc7d-d90f-4a98-84ae-fd65aff1f535 14 | fi 15 | echo 'Loading Linux 5.4.0-73-generic ...' 16 | linux /vmlinuz-ota.standby root=UUID=76543210-3210-3210-3210-ba9876543210 ro quiet splash $vt_handoff 17 | echo 'Loading initial ramdisk ...' 18 | initrd /initrd.img-ota.standby 19 | } 20 | -------------------------------------------------------------------------------- /src/otaclient/boot_control/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from .configs import BootloaderType 17 | from .protocol import BootControllerProtocol 18 | from .selecter import detect_bootloader, get_boot_controller 19 | 20 | __all__ = ( 21 | "get_boot_controller", 22 | "detect_bootloader", 23 | "BootloaderType", 24 | "BootControllerProtocol", 25 | ) 26 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_detect_ota_image_spec.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from hashlib import sha256 5 | 6 | from ota_metadata.utils.detect_ota_image_ver import check_if_ota_image_v1 7 | from otaclient_common.downloader import DownloaderPool 8 | from tests.conftest import cfg 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def test_detect_ota_image_ver(): 14 | _downloader = DownloaderPool(instance_num=3, hash_func=sha256) 15 | 16 | _check_res1 = check_if_ota_image_v1(cfg.OTA_IMAGE_URL, downloader_pool=_downloader) 17 | logger.info( 18 | f"check if {cfg.OTA_IMAGE_URL=} hosts OTA image v1: {_check_res1}(should be False)" 19 | ) 20 | assert not _check_res1 21 | 22 | _check_res2 = check_if_ota_image_v1( 23 | cfg.OTA_IMAGE_V1_URL, downloader_pool=_downloader 24 | ) 25 | logger.info( 26 | f"check if {cfg.OTA_IMAGE_V1_URL=} hosts OTA image v1: {_check_res2}(should be True)" 27 | ) 28 | assert _check_res2 29 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:best-practices" 5 | ], 6 | // allow sync_lockfile workflow to run without blocking renovate 7 | "gitIgnoredAuthors": [ 8 | "github-actions[bot]+sync-lockfile@users.noreply.github.com" 9 | ], 10 | "timezone": "Asia/Tokyo", 11 | // allow to run on non-office hours every day. 12 | // note that it is NOT the schedule of renovate task dispatching, 13 | // but the time window of allowing renovate to create PRs, 14 | // see https://docs.renovatebot.com/configuration-options/#schedule 15 | "schedule": ["* 20-23,0-8 * * 1-5", "* * * * 0,6"], 16 | "lockFileMaintenance": { 17 | "enabled": true 18 | }, 19 | "enabledManagers": [ 20 | "pep621" 21 | ], 22 | "packageRules": [ 23 | { 24 | "matchManagers": [ 25 | "pep621" 26 | ], 27 | "rangeStrategy": "widen" 28 | } 29 | ], 30 | "labels": [ 31 | "dependencies" 32 | ], 33 | "prHourlyLimit": 16 34 | } 35 | -------------------------------------------------------------------------------- /src/otaclient/ota_core/README.md: -------------------------------------------------------------------------------- 1 | # OTAClient core 2 | 3 | The implementation of OTA request and OTA execution handling. 4 | 5 | The structure of the `ota_core` package is as follows: 6 | 7 | 1. **`_updater_base`**: Implements the common shared base `OTAUpdateInitializer`, legacy OTA image support `LegacyOTAImageSupportMixin` and OTA image v1 support `OTAImageV1SupportMixin`. 8 | 9 | 2. **`_updater`**: Implements the complete OTA Update implementation, `OTAUpdaterForLegacyOTAImage` and `OTAUpdaterForOTAImageV1`. 10 | 11 | 3. **`_client_updater`**: Implements the dynamic OTAClient update as `OTAClientUpdate`. 12 | 13 | 4. **`_common`**: Common utilities and helper functions for `ota_core`. 14 | 15 | 5. **`_download_resources`**: Implements the downloading functionality for OTA operations as `DownloadHelper`. 16 | 17 | 6. **`_main`**: Implements the ota_process entrypoint and the RPC adapter between the otaclient gRPC server process and ota_core process. 18 | 19 | 7. **`_update_libs`**: Implements the OTA image spec version unspecific logics. 20 | -------------------------------------------------------------------------------- /docker/test_base/entry_point.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | TEST_ROOT=/test_root 5 | OTACLIENT_SRC=/otaclient_src 6 | OUTPUT_DIR="${OUTPUT_DIR:-/test_result}" 7 | 8 | mkdir -p ${TEST_ROOT} 9 | # source code needed to be rw, so copy it to the test_root 10 | cp -R ${OTACLIENT_SRC}/src ${TEST_ROOT} 11 | cp ${OTACLIENT_SRC}/uv.lock ${TEST_ROOT} 12 | # symlink all the other needed folders/files into test root 13 | ln -s ${OTACLIENT_SRC}/tests ${TEST_ROOT} 14 | ln -s ${OTACLIENT_SRC}/.git ${TEST_ROOT} 15 | ln -s ${OTACLIENT_SRC}/pyproject.toml ${TEST_ROOT} 16 | ln -s ${OTACLIENT_SRC}/README.md ${TEST_ROOT} 17 | ln -s ${OTACLIENT_SRC}/LICENSE.md ${TEST_ROOT} 18 | ln -s ${OTACLIENT_SRC}/scripts ${TEST_ROOT} || true 19 | 20 | # exec the input params 21 | echo "execute test with coverage" 22 | cd ${TEST_ROOT} 23 | uv run --python python3 --no-managed-python coverage run -m pytest --junit-xml=${OUTPUT_DIR}/pytest.xml ${@:-} 24 | uv run --python python3 --no-managed-python coverage combine 25 | uv run --python python3 --no-managed-python coverage xml -o ${OUTPUT_DIR}/coverage.xml 26 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2022 TIER IV, INC. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | This project uses open-source software, each under its own license. 16 | For details, see the table below: 17 | 18 | | Software | License | Source | 19 | |----------|---------------------------------------------------|-----------------------------------------------------| 20 | | certifi | [MPL-2.0](https://opensource.org/license/MPL-2.0) | [GitHub](https://github.com/certifi/python-certifi) | 21 | -------------------------------------------------------------------------------- /.github/actions/generate_manifest/action.yml: -------------------------------------------------------------------------------- 1 | name: "Generate Manifest" 2 | description: "Generate a manifest file for the OTA client" 3 | inputs: 4 | dir: 5 | description: "The directory containing the SquashFS and patches files" 6 | required: true 7 | output_manifest: 8 | description: "The output manifest file" 9 | required: true 10 | runs: 11 | using: "composite" 12 | steps: 13 | - name: setup python 14 | uses: actions/setup-python@v5 15 | with: 16 | python-version: 3.8 17 | 18 | - name: install build deps 19 | shell: bash 20 | run: | 21 | python -m pip install -U pip 22 | python -m pip install -U pydantic 23 | 24 | - name: generate json manifest 25 | env: 26 | DIR: ${{ inputs.dir }} 27 | OUTPUT_MANIFEST: ${{ inputs.output_manifest }} 28 | PYTHONPATH: ${{ github.workspace }}/src # to use otaclient_manifest module 29 | shell: bash 30 | run: | 31 | python ./.github/actions/generate_manifest/generate_manifest.py \ 32 | --dir ${DIR} \ 33 | --output ${OUTPUT_MANIFEST} 34 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | labels: 4 | - ignore-for-release 5 | authors: 6 | # not include the PRs for updating pre-commit-ci hooks ver 7 | - pre-commit-ci 8 | categories: 9 | - title: Security related! 10 | labels: 11 | - security 12 | - title: Breaking Changes! 13 | labels: 14 | - breaking-change 15 | - title: New Features 16 | labels: 17 | - feature 18 | - title: Bug Fixes 19 | labels: 20 | - bug 21 | - title: Improvements & Refinements 22 | labels: 23 | - refactor 24 | - refinement 25 | - title: Build, CI & Dependencies 26 | labels: 27 | - build/ci 28 | - dependencies 29 | - title: Tests Updates 30 | labels: 31 | - test_files 32 | - title: Docs Updates 33 | labels: 34 | - documentation 35 | - title: Tools Updates 36 | labels: 37 | - tools 38 | - title: Chore & Misc. 39 | labels: 40 | - chore 41 | - misc 42 | - title: Other Changes 43 | labels: 44 | - "*" 45 | -------------------------------------------------------------------------------- /tests/data/extlinux.conf-r35.4.1-template2: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT primary 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 11 | APPEND ${cbootargs} root=PARTUUID= rw rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 12 | 13 | # When testing a custom kernel, it is recommended that you create a backup of 14 | # the original kernel and add a new entry to this file so that the device can 15 | # fallback to the original kernel. To do this: 16 | # 17 | # 1, Make a backup of the original kernel 18 | # sudo cp /boot/Image /boot/Image.backup 19 | # 20 | # 2, Copy your custom kernel into /boot/Image 21 | # 22 | # 3, Uncomment below menu setting lines for the original kernel 23 | # 24 | # 4, Reboot 25 | 26 | # LABEL backup 27 | # MENU LABEL backup kernel 28 | # LINUX /boot/Image.backup 29 | # INITRD /boot/initrd 30 | # APPEND ${cbootargs} 31 | -------------------------------------------------------------------------------- /docker/test_base/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG UBUNTU_BASE 2 | 3 | FROM ghcr.io/tier4/ota-client/ota_img_for_test:ubuntu_22.04 AS ota_image 4 | 5 | FROM ghcr.io/tier4/ota-client/ota_img_for_test:ubuntu_22.04-ota_image_v1 AS ota_image_v1 6 | 7 | FROM ${UBUNTU_BASE} 8 | 9 | ARG UV_VERSION=0.8.22 10 | 11 | # NOTE: for fallback when no entry_point.sh is specified in compose file 12 | COPY --chmod=755 ./entry_point.sh /entry_point.sh 13 | COPY --from=ota_image /ota-image /ota-image 14 | COPY --from=ota_image /certs /certs 15 | COPY --from=ota_image_v1 /ota-image /ota-image_v1 16 | COPY --from=ota_image_v1 /certs /certs_ota-image_v1 17 | 18 | # bootstrapping the python environment 19 | RUN set -eux; \ 20 | apt-get update; \ 21 | apt-get install -y -qq --no-install-recommends \ 22 | git ca-certificates wget python3 python3-setuptools; \ 23 | export UV_INSTALL_DIR=/usr/bin; \ 24 | wget -qO- "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh; \ 25 | apt-get clean; \ 26 | rm -rf \ 27 | /tmp/* \ 28 | /var/lib/apt/lists/* \ 29 | /var/tmp/* 30 | 31 | ENTRYPOINT [ "/bin/bash", "/entry_point.sh" ] 32 | -------------------------------------------------------------------------------- /src/otaclient/ota_core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ._client_updater import OTAClientUpdater 16 | from ._main import OTAClient, ota_core_process 17 | from ._updater import OTAUpdaterForLegacyOTAImage, OTAUpdaterForOTAImageV1 18 | from ._updater_base import OTAUpdateInitializer 19 | 20 | __all__ = [ 21 | "ota_core_process", 22 | "OTAClient", 23 | "OTAUpdateInitializer", 24 | "OTAUpdaterForLegacyOTAImage", 25 | "OTAUpdaterForOTAImageV1", 26 | "OTAClientUpdater", 27 | ] 28 | -------------------------------------------------------------------------------- /src/otaclient_common/download_info.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from dataclasses import dataclass 17 | from pathlib import Path 18 | from typing import Optional 19 | 20 | 21 | @dataclass 22 | class DownloadInfo: 23 | url: str 24 | dst: Path 25 | original_size: int = 0 26 | """NOTE: we are using transparent decompression, so we always use the original_size.""" 27 | digest_alg: Optional[str] = None 28 | digest: Optional[str] = None 29 | compression_alg: Optional[str] = None 30 | -------------------------------------------------------------------------------- /src/ota_metadata/file_table/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FILE_TABLE_MEDIA_TYPE = ( 16 | "application/vnd.tier4.ota.file-based-ota-image.file_table.v1.sqlite3" 17 | ) 18 | """The supported file_table version.""" 19 | 20 | FILE_TABLE_FNAME = "file_table.sqlite3" 21 | MEDIA_TYPE_FNAME = "mediaType" 22 | 23 | FT_REGULAR_TABLE_NAME = "ft_regular" 24 | FT_NON_REGULAR_TABLE_NAME = "ft_non_regular" 25 | FT_DIR_TABLE_NAME = "ft_dir" 26 | FT_INODE_TABLE_NAME = "ft_inode" 27 | FT_RESOURCE_TABLE_NAME = "ft_resource" 28 | -------------------------------------------------------------------------------- /tests/data/extlinux.conf-r35.4.1-updated2: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT primary 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 11 | APPEND ${cbootargs} root=PARTUUID=11aa-bbcc-22dd rw rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 12 | 13 | # When testing a custom kernel, it is recommended that you create a backup of 14 | # the original kernel and add a new entry to this file so that the device can 15 | # fallback to the original kernel. To do this: 16 | # 17 | # 1, Make a backup of the original kernel 18 | # sudo cp /boot/Image /boot/Image.backup 19 | # 20 | # 2, Copy your custom kernel into /boot/Image 21 | # 22 | # 3, Uncomment below menu setting lines for the original kernel 23 | # 24 | # 4, Reboot 25 | 26 | # LABEL backup 27 | # MENU LABEL backup kernel 28 | # LINUX /boot/Image.backup 29 | # INITRD /boot/initrd 30 | # APPEND ${cbootargs} 31 | -------------------------------------------------------------------------------- /docker/test_base/entry_point_py313.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | PY_VER=python3.13 5 | TEST_ROOT=/test_root 6 | OTACLIENT_SRC=/otaclient_src 7 | OUTPUT_DIR="${OUTPUT_DIR:-/test_result}" 8 | 9 | mkdir -p "${TEST_ROOT}" 10 | # source code needed to be rw, so copy it to the test_root 11 | cp -R "${OTACLIENT_SRC}/src" "${TEST_ROOT}" 12 | cp "${OTACLIENT_SRC}/uv.lock" "${TEST_ROOT}" 13 | # symlink all the other needed folders/files into test root 14 | ln -sf "${OTACLIENT_SRC}/tests" "${TEST_ROOT}" 15 | ln -sf "${OTACLIENT_SRC}/.git" "${TEST_ROOT}" 16 | ln -sf "${OTACLIENT_SRC}/pyproject.toml" "${TEST_ROOT}" 17 | ln -sf "${OTACLIENT_SRC}/README.md" "${TEST_ROOT}" 18 | ln -sf "${OTACLIENT_SRC}/LICENSE.md" "${TEST_ROOT}" 19 | ln -sf "${OTACLIENT_SRC}/scripts" "${TEST_ROOT}" 20 | 21 | # exec the input params 22 | echo "execute test with coverage" 23 | cd "${TEST_ROOT}" 24 | # NOTE: use managed python3.13 provided by uv 25 | uv run --python "${PY_VER}" coverage run -m pytest --junit-xml="${OUTPUT_DIR}/pytest.xml" "$@" 26 | uv run --python "${PY_VER}" coverage combine 27 | uv run --python "${PY_VER}" coverage xml -o "${OUTPUT_DIR}/coverage.xml" 28 | -------------------------------------------------------------------------------- /src/otaclient/create_standby/_common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | from otaclient_common.thread_safe_container import ShardedThreadSafeDict 18 | 19 | 20 | class ResourcesDigestWithSize(ShardedThreadSafeDict[bytes, int]): 21 | # NOTE: sha256 digest is already unique, no need to do hash again, 22 | # just directly calculate shard index from its value. 23 | def _shard_index(self, key: bytes) -> int: 24 | return int.from_bytes(key, byteorder="big") % self._num_of_shards 25 | -------------------------------------------------------------------------------- /docker/mini_ota_img/sys_img.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG UBUNTU_BASE=ubuntu:22.04 2 | 3 | FROM ${UBUNTU_BASE} 4 | 5 | ARG UBUNTU_BASE 6 | 7 | SHELL ["/bin/bash", "-c"] 8 | ENV DEBIAN_FRONTEND=noninteractive 9 | ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" 10 | # NOTE(20250225): for PR#492, add a folder that contains 5000 empty files 11 | ENV EMPTY_FILE_FOLDER="/usr/empty_files" 12 | ENV EMPTY_FILES_COUNT=1024 13 | 14 | # special treatment to the ota-image: create file that needs url escaping 15 | # NOTE: include special identifiers #?; into the pathname 16 | RUN set -eux; \ 17 | echo -n "${SPECIAL_FILE}" > "/${SPECIAL_FILE}"; \ 18 | # create special folder containing a lot of empty files 19 | mkdir -p ${EMPTY_FILE_FOLDER}; for i in $(seq 1 ${EMPTY_FILES_COUNT}); do touch "${EMPTY_FILE_FOLDER}/$i"; done; \ 20 | # install required packages 21 | apt-get update -qq; \ 22 | apt-get install -y linux-image-generic; \ 23 | apt-get clean; \ 24 | rm -rf \ 25 | /tmp/* \ 26 | /var/lib/apt/lists/* \ 27 | /var/tmp/* 28 | 29 | LABEL org.opencontainers.image.description="A mini system image for OTAClient test, based on ${UBUNTU_BASE}" 30 | 31 | CMD ["/bin/bash"] 32 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/refinement.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 7 | 8 | ## Check list 9 | 10 | 11 | 12 | - [ ] test file(s) that cover the change(s) are implemented. 13 | - [ ] local tests are passing. 14 | 15 | ## Changes 16 | 17 | 18 | 19 | ## Behavior changes 20 | 21 | Does this PR introduce behavior change(s)? 22 | 23 | - [ ] Yes, internal behavior (will not impact user experience). 24 | - [ ] Yes, external behavior (will impact user experience). 25 | - [ ] No. 26 | 27 | ### Previous behavior 28 | 29 | 30 | 31 | ### Behavior with this PR 32 | 33 | 34 | 35 | ## Breaking change 36 | 37 | Does this PR introduce breaking change(s)? 38 | 39 | - [ ] Yes. 40 | - [ ] No. 41 | 42 | 43 | 44 | ## Related links & tickets 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/otaclient/create_standby/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from otaclient.create_standby.delta_gen import ( 17 | InPlaceDeltaGenFullDiskScan, 18 | InPlaceDeltaWithBaseFileTable, 19 | RebuildDeltaGenFullDiskScan, 20 | RebuildDeltaWithBaseFileTable, 21 | ) 22 | from otaclient.create_standby.update_slot import UpdateStandbySlot 23 | from otaclient.create_standby.utils import can_use_in_place_mode 24 | 25 | __all__ = [ 26 | "can_use_in_place_mode", 27 | "UpdateStandbySlot", 28 | "InPlaceDeltaGenFullDiskScan", 29 | "InPlaceDeltaWithBaseFileTable", 30 | "RebuildDeltaGenFullDiskScan", 31 | "RebuildDeltaWithBaseFileTable", 32 | ] 33 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_configs/test_cfg_consts.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from pytest_mock import MockerFixture 19 | 20 | from otaclient.configs import Consts, _cfg_consts, dynamic_root 21 | 22 | 23 | def test_cfg_consts_dynamic_root(mocker: MockerFixture): 24 | _real_root = "/host_root" 25 | mocked_consts = Consts() 26 | mocker.patch.object(mocked_consts, "_ACTIVE_ROOT", _real_root) 27 | mocker.patch.object(_cfg_consts, "cfg_consts", mocked_consts) 28 | 29 | assert mocked_consts.ACTIVE_ROOT == _real_root 30 | assert dynamic_root(mocked_consts.BOOT_DPATH) == "/host_root/boot" 31 | assert mocked_consts.OTA_API_SERVER_PORT == 50051 32 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: '^$|.*_pb2(_grpc)?\.py' 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v6.0.0 5 | hooks: 6 | - id: check-yaml 7 | args: ["--allow-multiple-documents"] 8 | - id: check-toml 9 | - id: end-of-file-fixer 10 | exclude: | 11 | (?x)( 12 | .*\.squashfs$| 13 | .*\.patch$ 14 | ) 15 | - id: trailing-whitespace 16 | - repo: https://github.com/astral-sh/ruff-pre-commit 17 | rev: v0.14.7 18 | hooks: 19 | - id: ruff 20 | args: [--fix] 21 | # Using this mirror lets us use mypyc-compiled black, which is about 2x faster 22 | - repo: https://github.com/psf/black-pre-commit-mirror 23 | rev: 25.11.0 24 | hooks: 25 | - id: black 26 | # It is recommended to specify the latest version of Python 27 | # supported by your project here, or alternatively use 28 | # pre-commit's default_language_version, see 29 | # https://pre-commit.com/#top_level-default_language_version 30 | language_version: python3.11 31 | - repo: https://github.com/igorshubovych/markdownlint-cli 32 | rev: v0.46.0 33 | hooks: 34 | - id: markdownlint 35 | args: ["-c", ".markdownlint.yaml", "--fix"] 36 | ci: 37 | autoupdate_schedule: monthly 38 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/feature.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 8 | 9 | ## Check list 10 | 11 | 12 | 13 | - [ ] test file(s) that cover the change(s) are implemented. 14 | - [ ] local tests are passing. 15 | - [ ] design docs/implementation docs are prepared. 16 | 17 | ## Documents 18 | 19 | 20 | 21 | ## Changes 22 | 23 | 24 | 25 | ## Behavior changes 26 | 27 | Does this PR introduce behavior change(s)? 28 | 29 | - [ ] Yes, internal behavior (will not impact user experience). 30 | - [ ] Yes, external behavior (will impact user experience). 31 | - [ ] No. 32 | 33 | ### Previous behavior 34 | 35 | 36 | 37 | ### Behavior with this PR 38 | 39 | 40 | 41 | ## Breaking change 42 | 43 | Does this PR introduce breaking change(s)? 44 | 45 | - [ ] Yes. 46 | - [ ] No. 47 | 48 | 49 | 50 | ## Related links & tickets 51 | 52 | 53 | -------------------------------------------------------------------------------- /docker/test_base/README.md: -------------------------------------------------------------------------------- 1 | # Base container image for OTAClient test 2 | 3 | Built images and corresponding Ubuntu version: 4 | 5 | 1. `ubuntu:20.04`: `ghcr.io/tier4/ota-client/test_base:ubuntu_20.04` 6 | 1. `ubuntu:22.04`: `ghcr.io/tier4/ota-client/test_base:ubuntu_22.04` 7 | 1. `ubuntu:24.04`: `ghcr.io/tier4/ota-client/test_base:ubuntu_24.04` 8 | 9 | ## Build cmds 10 | 11 | ### GitHub Actions build workflow 12 | 13 | Images are automatically built and pushed to ghcr.io when changes are made to `docker/test_base/` directory. See [`.github/workflows/test_image.yaml`](../../.github/workflows/test_image.yaml) for the workflow configuration. 14 | 15 | To manually trigger a build, use the workflow dispatch feature on GitHub Actions. 16 | 17 | ### Manual build command example 18 | 19 | ```shell 20 | BASE_URI=ghcr.io/tier4/ota-client/test_base 21 | UBUNTU_VER=20.04 22 | docker login ghcr.io -u YOUR_GITHUB_USERNAME -p YOUR_GITHUB_TOKEN 23 | docker buildx create --name zstd-builder --driver docker-container --use 24 | docker buildx build --builder zstd-builder \ 25 | -t ghcr.io/tier4/ota-client/test_base:ubuntu_${UBUNTU_VER} \ 26 | --build-arg=UBUNTU_BASE=ubuntu:${UBUNTU_VER} \ 27 | --output type=image,name=${BASE_URI}:ubuntu_${UBUNTU_VER},compression=zstd,compression-level=19,oci-mediatypes=true,force-compression=true,push=true \ 28 | . 29 | ``` 30 | -------------------------------------------------------------------------------- /src/otaclient_manifest/schema.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Schema definition for manifest.json.""" 15 | 16 | from datetime import datetime 17 | from typing import List, Literal, Optional 18 | 19 | from pydantic import BaseModel 20 | 21 | 22 | class PackageExtraMetadata(BaseModel): 23 | patch_base_version: Optional[str] = None 24 | 25 | 26 | class ReleasePackage(BaseModel): 27 | filename: str 28 | version: str 29 | type: Literal["squashfs", "patch"] 30 | architecture: Literal["x86_64", "arm64"] 31 | size: int 32 | checksum: str 33 | metadata: Optional[PackageExtraMetadata] = None 34 | 35 | 36 | class Manifest(BaseModel): 37 | schema_version: str = "1" 38 | date: datetime 39 | packages: List[ReleasePackage] 40 | -------------------------------------------------------------------------------- /src/otaclient/configs/_common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from pydantic import BaseModel, ConfigDict 19 | from pydantic_settings import BaseSettings, SettingsConfigDict 20 | 21 | # prefix for environmental vars name for configs. 22 | ENV_PREFIX = "OTA_" 23 | 24 | 25 | class BaseConfigurableConfig(BaseSettings): 26 | """Common base for configs that are configurable via ENV.""" 27 | 28 | model_config = SettingsConfigDict( 29 | env_prefix=ENV_PREFIX, 30 | frozen=True, 31 | validate_default=True, 32 | ) 33 | 34 | 35 | class BaseFixedConfig(BaseModel): 36 | """Common base for configs that should be fixed and not changable.""" 37 | 38 | model_config = ConfigDict(frozen=True, validate_default=True) 39 | -------------------------------------------------------------------------------- /tests/data/ota_image_builder/full_annotations.yaml: -------------------------------------------------------------------------------- 1 | # Example annotations for build a full OTA image targetting one ECU. 2 | vnd.tier4.ota.ota-image-builder.version: "1.0.0" 3 | 4 | vnd.tier4.pilot-auto.platform: "example-platform" 5 | vnd.tier4.pilot-auto.project.source-repo: "https://github.com/tier4/ota-image-libs" 6 | vnd.tier4.pilot-auto.project.version: "1.0.0" 7 | vnd.tier4.pilot-auto.project.release-commit: "abcdef1234567890" 8 | vnd.tier4.pilot-auto.project.release-branch: "main" 9 | 10 | vnd.tier4.ota.release-key: "dev" 11 | vnd.tier4.web-auto.project: "some-project" 12 | vnd.tier4.web-auto.project.id: "some-project-id" 13 | vnd.tier4.web-auto.catalog: "some-catalog" 14 | vnd.tier4.web-auto.catalog.id: "some-catalog-id" 15 | vnd.tier4.web-auto.env: "dev" 16 | vnd.tier4.web-auto.cicd.release-id: "693ac51b-5c23-4d07-b0de-6fcf896b15b9" 17 | vnd.tier4.web-auto.cicd.release-name: "20250926123456-beta/ota" 18 | 19 | # for image-manifest 20 | vnd.tier4.pilot-auto.platform.ecu.hardware-model: "example-hardware-model" 21 | vnd.tier4.pilot-auto.platform.ecu.hardware-series: "example-hardware-series" 22 | vnd.tier4.pilot-auto.platform.ecu.architecture: "x86_64" 23 | 24 | # for image-config 25 | vnd.tier4.image.base-image: "ubuntu:22.04" 26 | vnd.tier4.image.os: "Ubuntu" 27 | vnd.tier4.image.os.version: "22.04" 28 | description: "Example OTA image with annotations for add-image cmd" 29 | created: "2025-07-15T15:43:32Z" 30 | -------------------------------------------------------------------------------- /.github/workflows/issue-metrics.yml: -------------------------------------------------------------------------------- 1 | name: Monthly issue metrics 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: "3 2 1 * *" 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | build: 12 | name: issue metrics 13 | runs-on: ubuntu-latest 14 | permissions: 15 | issues: write 16 | pull-requests: read 17 | steps: 18 | - name: Get dates for last month 19 | shell: bash 20 | run: | 21 | # Calculate the first day of the previous month 22 | first_day=$(date -d "last month" +%Y-%m-01) 23 | 24 | # Calculate the last day of the previous month 25 | last_day=$(date -d "$first_day +1 month -1 day" +%Y-%m-%d) 26 | 27 | #Set an environment variable with the date range 28 | echo "$first_day..$last_day" 29 | echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" 30 | 31 | - name: Run issue-metrics tool 32 | uses: github/issue-metrics@v3 33 | env: 34 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | SEARCH_QUERY: 'repo:tier4/ota-client is:pr created:${{ env.last_month }} -reason:"not planned"' 36 | 37 | - name: Create issue 38 | uses: peter-evans/create-issue-from-file@v6 39 | with: 40 | title: Monthly issue metrics report 41 | token: ${{ secrets.GITHUB_TOKEN }} 42 | content-filepath: ./issue_metrics.md 43 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_proto_wrapper/example.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2022 TIER IV, INC. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | 16 | syntax = "proto3"; 17 | 18 | import "google/protobuf/duration.proto"; 19 | 20 | enum SampleEnum { 21 | VALUE_0 = 0; 22 | VALUE_1 = 1; 23 | VALUE_2 = 2; 24 | } 25 | 26 | message InnerMessage { 27 | uint32 int_field = 1; 28 | double double_field = 2; 29 | string str_field = 3; 30 | google.protobuf.Duration duration_field = 4; 31 | SampleEnum enum_field = 5; 32 | } 33 | 34 | message OuterMessage { 35 | repeated string repeated_scalar_field = 1; 36 | repeated InnerMessage repeated_composite_field = 2; 37 | InnerMessage nested_msg = 3; 38 | map mapping_scalar_field = 4; 39 | map mapping_composite_field = 5; 40 | } 41 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/default_grub: -------------------------------------------------------------------------------- 1 | # If you change this file, run 'update-grub' afterwards to update 2 | # /boot/grub/grub.cfg. 3 | # For full documentation of the options in this file, see: 4 | # info -f grub -n 'Simple configuration' 5 | 6 | GRUB_DEFAULT=1 7 | GRUB_TIMEOUT_STYLE=menu 8 | GRUB_TIMEOUT=10 9 | GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` 10 | GRUB_CMDLINE_LINUX_DEFAULT="quiet splash pcie_aspm=off" 11 | GRUB_CMDLINE_LINUX="" 12 | 13 | # Uncomment to enable BadRAM filtering, modify to suit your needs 14 | # This works with Linux (no patch required) and with any kernel that obtains 15 | # the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) 16 | #GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" 17 | 18 | # Uncomment to disable graphical terminal (grub-pc only) 19 | #GRUB_TERMINAL=console 20 | 21 | # The resolution used on graphical terminal 22 | # note that you can use only modes which your graphic card supports via VBE 23 | # you can see them in real GRUB with the command `vbeinfo' 24 | #GRUB_GFXMODE=640x480 25 | 26 | # Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux 27 | #GRUB_DISABLE_LINUX_UUID=true 28 | 29 | # Uncomment to disable generation of recovery mode menu entries 30 | #GRUB_DISABLE_RECOVERY="true" 31 | 32 | # Uncomment to get a beep at grub start 33 | #GRUB_INIT_TUNE="480 440 1" 34 | GRUB_DISABLE_SUBMENU=y 35 | GRUB_DISABLE_OS_PROBER=true 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OTAClient 2 | 3 | ## Overview 4 | 5 | OTAClient is software to perform over-the-air software updates for linux devices. 6 | It provides a set of APIs for user to start the OTA and monitor the progress and status. 7 | 8 | It is designed to work with web.auto FMS OTA component. 9 | 10 | ## Feature 11 | 12 | - A/B partition update with support for generic x86_64 device, NVIDIA Jetson series based devices and Raspberry Pi 13 | device. 14 | - Full Rootfs update, with delta update support. 15 | - Local delta calculation, allowing update to any version of OTA image without the need of a pre-generated delta OTA 16 | package. 17 | - Support persist files from active slot to newly updated slot. 18 | - Verification over OTA image by digital signature and PKI. 19 | - Support for protected OTA server with cookie. 20 | - Optional OTA proxy support and OTA cache support. 21 | - Multiple ECU OTA supports. 22 | 23 | ## License 24 | 25 | OTAClient is licensed under the Apache License 2.0. 26 | 27 | This project uses open-source software, each under its own license. 28 | For details, see the table below: 29 | 30 | | Software | License | Source | 31 | |----------|---------------------------------------------------|-----------------------------------------------------| 32 | | certifi | [MPL-2.0](https://opensource.org/license/MPL-2.0) | [GitHub](https://github.com/certifi/python-certifi) | 33 | -------------------------------------------------------------------------------- /docker/app_img/README.md: -------------------------------------------------------------------------------- 1 | ## otaclient application squashfs image creation howto 2 | 3 | ### Build otaclient APP container image 4 | 5 | ```bash 6 | # at the project root 7 | sudo docker build \ 8 | -f docker/app_img/Dockerfile \ 9 | --no-cache \ 10 | --build-arg=UBUNTU_BASE=ubuntu:22.04 \ 11 | --build-arg=OTACLIENT_VERSION=${OTACLIENT_VERSION} \ 12 | -t otaclient_app:${OTACLIENT_VERSION} . 13 | ``` 14 | 15 | ### Build otaclient APP squashfs image from APP container image 16 | 17 | ```bash 18 | # at the project root 19 | sudo docker create --name otaclient_app_export otaclient_app:${OTACLIENT_VERSION} 20 | sudo docker export otaclient_app_export | mksquashfs - dist/otaclient_${OTACLIENT_VERSION}.squashfs \ 21 | -tar -b 1M \ 22 | -mkfs-time 1729810800 \ 23 | -all-time 1729810800 \ 24 | -no-xattrs \ 25 | -all-root \ 26 | -progress \ 27 | -comp gzip 28 | # if squashfs zstd kernel support is available, use the following instead of gzip 29 | # -comp zstd \ 30 | # -Xcompression-level 22 31 | ``` 32 | 33 | ## Step to create otaclient app image update patch 34 | 35 | ```bash 36 | zstd --patch-from=otaclient_${BASE_VERSION}.squashfs otaclient_${TARGET_VERSION}.squashfs -o ${BASE_VERSION}-${TARGET_VERSION}_patch 37 | ``` 38 | 39 | ## Step to apply app image update patch 40 | 41 | ```bash 42 | zstd -d --patch-from=otaclient_${BASE_VERSION}.squashfs ${BASE_VERSION}-${TARGET_VERSION}_patch -o otaclient_${TARGET_VERSION}.squashfs 43 | ``` 44 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_ota_core/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import pytest 19 | import pytest_mock 20 | 21 | from otaclient import ota_core 22 | from tests.conftest import TestConfiguration as cfg 23 | 24 | OTA_UPDATER_MODULE = ota_core._updater.__name__ 25 | 26 | 27 | @pytest.fixture(autouse=True, scope="module") 28 | def mock_certs_dir(module_mocker: pytest_mock.MockerFixture): 29 | """Mock to use the certs from the OTA test base image.""" 30 | from otaclient.configs.cfg import cfg as _cfg 31 | 32 | module_mocker.patch.object( 33 | _cfg, 34 | "CERT_DPATH", 35 | cfg.CERTS_DIR, 36 | ) 37 | 38 | 39 | @pytest.fixture(autouse=True, scope="module") 40 | def module_scope_mock(module_mocker: pytest_mock.MockerFixture): 41 | module_mocker.patch( 42 | f"{OTA_UPDATER_MODULE}.fstrim_at_subprocess", 43 | module_mocker.MagicMock(), 44 | ) 45 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu 3 | { 4 | "name": "otaclient dev environment - Python 3.8", 5 | // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile 6 | "image": "mcr.microsoft.com/devcontainers/base:jammy", 7 | 8 | // Features to add to the dev container. More info: https://containers.dev/features. 9 | "features": { 10 | // for running tests with docker inside devcontainer 11 | "ghcr.io/devcontainers/features/docker-in-docker:2": { }, 12 | "ghcr.io/devcontainers/features/git:1": { }, 13 | "ghcr.io/devcontainers/features/python:1": { "version": "3.8", "toolsToInstall": "uv,ruff" } 14 | 15 | }, 16 | 17 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 18 | // "forwardPorts": [], 19 | 20 | // Use 'postCreateCommand' to run commands after the container is created. 21 | // bootstrap venv with uv 22 | "postCreateCommand": "uv sync", 23 | 24 | // Configure tool-specific properties. 25 | "customizations": { 26 | // minimum set of extensions for python developing 27 | "vscode": { 28 | "extensions": [ 29 | "ms-python.python", 30 | "ms-python.vscode-pylance", 31 | "ms-python.debugpy", 32 | "charliermarsh.ruff", 33 | "redhat.vscode-yaml" 34 | ] 35 | } 36 | } 37 | 38 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 39 | // "remoteUser": "root" 40 | } 41 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/extlinux.conf_slot_a: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT Tier4 ISX021 GMSL2 Camera Device Tree Overlay 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra194-rqx-58g.dtb 11 | APPEND ${cbootargs} quiet root=PARTUUID=aaaaaaaa-0000-0000-0000-aaaaaaaaaaaa rw rootwait rootfstype=ext4 console=ttyTCU0,115200n8 console=tty0 fbcon=map:0 net.ifnames=0 rootfstype=ext4 12 | 13 | # When testing a custom kernel, it is recommended that you create a backup of 14 | # the original kernel and add a new entry to this file so that the device can 15 | # fallback to the original kernel. To do this: 16 | # 17 | # 1, Make a backup of the original kernel 18 | # sudo cp /boot/Image /boot/Image.backup 19 | # 20 | # 2, Copy your custom kernel into /boot/Image 21 | # 22 | # 3, Uncomment below menu setting lines for the original kernel 23 | # 24 | # 4, Reboot 25 | 26 | # LABEL backup 27 | # MENU LABEL backup kernel 28 | # LINUX /boot/Image.backup 29 | # INITRD /boot/initrd 30 | # APPEND ${cbootargs} 31 | 32 | LABEL Tier4 ISX021 GMSL2 Camera Device Tree Overlay 33 | MENU LABEL Tier4 ISX021 GMSL2 Camera Device Tree Overlay 34 | LINUX /boot/Image 35 | FDT /boot/kernel_tegra194-rqx-58g-tier4-isx021-gmsl2-camera-device-tree-overlay.dtb 36 | INITRD /boot/initrd 37 | APPEND ${cbootargs} quiet root=PARTUUID=aaaaaaaa-0000-0000-0000-aaaaaaaaaaaa rw rootwait rootfstype=ext4 console=ttyTCU0,115200n8 console=tty0 fbcon=map:0 net.ifnames=0 rootfstype=ext4 38 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_boot_control/extlinux.conf_slot_b: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT Tier4 ISX021 GMSL2 Camera Device Tree Overlay 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra194-rqx-58g.dtb 11 | APPEND ${cbootargs} quiet root=PARTUUID=bbbbbbbb-1111-1111-1111-bbbbbbbbbbbb rw rootwait rootfstype=ext4 console=ttyTCU0,115200n8 console=tty0 fbcon=map:0 net.ifnames=0 rootfstype=ext4 12 | 13 | # When testing a custom kernel, it is recommended that you create a backup of 14 | # the original kernel and add a new entry to this file so that the device can 15 | # fallback to the original kernel. To do this: 16 | # 17 | # 1, Make a backup of the original kernel 18 | # sudo cp /boot/Image /boot/Image.backup 19 | # 20 | # 2, Copy your custom kernel into /boot/Image 21 | # 22 | # 3, Uncomment below menu setting lines for the original kernel 23 | # 24 | # 4, Reboot 25 | 26 | # LABEL backup 27 | # MENU LABEL backup kernel 28 | # LINUX /boot/Image.backup 29 | # INITRD /boot/initrd 30 | # APPEND ${cbootargs} 31 | 32 | LABEL Tier4 ISX021 GMSL2 Camera Device Tree Overlay 33 | MENU LABEL Tier4 ISX021 GMSL2 Camera Device Tree Overlay 34 | LINUX /boot/Image 35 | FDT /boot/kernel_tegra194-rqx-58g-tier4-isx021-gmsl2-camera-device-tree-overlay.dtb 36 | INITRD /boot/initrd 37 | APPEND ${cbootargs} quiet root=PARTUUID=bbbbbbbb-1111-1111-1111-bbbbbbbbbbbb rw rootwait rootfstype=ext4 console=ttyTCU0,115200n8 console=tty0 fbcon=map:0 net.ifnames=0 rootfstype=ext4 38 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_typing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from pathlib import Path 19 | from subprocess import check_output 20 | 21 | from otaclient_common._typing import StrEnum 22 | 23 | 24 | class EnumForTest(StrEnum): 25 | A = "A" 26 | 27 | 28 | def test_str_enum(tmp_path: Path): 29 | # str enum should be able to compare with string instance directly. 30 | assert EnumForTest.A == EnumForTest.A.value 31 | # str enum's __format__ should be the str's one, returning the str value. 32 | assert f"{EnumForTest.A}" == EnumForTest.A.value 33 | # our version of str enum for < 3.11 fully aligns with >= 3.11, which __str__ 34 | # is the str type's one. 35 | assert str(EnumForTest.A) == EnumForTest.A.value 36 | 37 | # When directly use as str, StrEnum should be behaved like a normal string. 38 | test_file = tmp_path / "file_written" 39 | test_file.write_text(EnumForTest.A) 40 | 41 | assert ( 42 | check_output(["cat", str(test_file)]).decode() 43 | == EnumForTest.A 44 | == EnumForTest.A.value 45 | ) 46 | -------------------------------------------------------------------------------- /tools/status_monitor/configs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class BasicConfig: 17 | # ------ Main window configs ------ # 18 | PAD_ILNIT_HLINES = 600 19 | MAINWIN_TITLE_HLINES = 1 20 | MAINWIN_MANUAL_HLINES = 2 21 | MAINWIN_LEFT_RIGHT_GAP = 1 22 | MIN_TERMINAL_SIZE = (10, 30) 23 | # NOTE: due to number keys' limitation, only 10 ECUs are allowed in total 24 | MAX_ECU_ALLOWED = 10 25 | MAINWIN_BOXES_PER_ROW = 2 26 | MAINWIN_BOXES_ROW_MAX = MAX_ECU_ALLOWED // MAINWIN_BOXES_PER_ROW 27 | 28 | # how many lines to go up/down when scroll 29 | SCROLL_LINES = 6 30 | # the interval to handle input and refresh the display, 31 | # should not be larger than 1, otherwise the window will be 32 | # very lag and broken on re-size. 33 | RENDER_INTERVAL = 0.01 34 | 35 | # ------ ECU status box configs ------ # 36 | ECU_DISPLAY_BOX_HLINES = 14 37 | ECU_DISPLAY_BOX_HCOLS = 60 38 | 39 | 40 | class KeyMapping: 41 | EXIT_ECU_STATUS_BOX_SUBWIN = ord("x") 42 | PAUSE = ord("p") 43 | ALT_OR_ESC = 27 44 | 45 | 46 | config = BasicConfig() 47 | key_mapping = KeyMapping() 48 | -------------------------------------------------------------------------------- /src/ota_metadata/utils/detect_ota_image_ver.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import time 5 | from http import HTTPStatus 6 | from urllib.parse import urlsplit 7 | 8 | from otaclient_common.downloader import DownloaderPool 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | OTA_IMAGE_V1_HINT = "oci-layout" 13 | DOWNLOAD_TIMEOUT = 2 # hint file only takes 31bytes 14 | RETRY_TIMES = 12 15 | RETRY_INTERVAL = 1 # second 16 | 17 | 18 | def check_if_ota_image_v1(base_url: str, *, downloader_pool: DownloaderPool) -> bool: 19 | """Determine whether an OTA image host is OTA Image v1.""" 20 | _downloader = downloader_pool.get_instance() 21 | try: 22 | _hint_file_url = f"{base_url.rstrip('/')}/{OTA_IMAGE_V1_HINT}" 23 | if _downloader._force_http: 24 | _hint_file_url = urlsplit(_hint_file_url)._replace(scheme="http").geturl() 25 | 26 | for _ in range(RETRY_TIMES): 27 | try: 28 | resp = _downloader._session.get( 29 | _hint_file_url, timeout=DOWNLOAD_TIMEOUT 30 | ) 31 | _status_code = resp.status_code 32 | if _status_code == HTTPStatus.OK: 33 | return True 34 | # NOTE(20251010): note that cloudfront endpoint will return 403 on non-existed path. 35 | if _status_code in [HTTPStatus.UNAUTHORIZED, HTTPStatus.NOT_FOUND]: 36 | return False 37 | except Exception: 38 | pass 39 | 40 | time.sleep(RETRY_INTERVAL) 41 | return False 42 | except Exception as e: 43 | logger.warning(f"unexpected failure during probing image version: {e}") 44 | return False 45 | finally: 46 | downloader_pool.release_instance() 47 | -------------------------------------------------------------------------------- /tools/status_monitor/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import argparse 17 | import curses 18 | import sys 19 | 20 | from .ecu_status_tracker import Tracker 21 | from .main_win import MainScreen 22 | 23 | 24 | def main(title: str, host: str, port: int): 25 | tracker = Tracker() 26 | tracker.start(host, port) 27 | 28 | main_scrn = MainScreen(title, display_boxes_getter=tracker.get_display_boxes) 29 | try: 30 | curses.wrapper(main_scrn.main) 31 | except KeyboardInterrupt: 32 | print("stop OTA status monitor") 33 | sys.exit(0) 34 | 35 | 36 | if __name__ == "__main__": 37 | parser = argparse.ArgumentParser( 38 | prog="ota_status_monitor", 39 | description="CLI program for monitoring target ecu status", 40 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 41 | ) 42 | parser.add_argument("--host", help="server listen ip", default="192.168.10.11") 43 | parser.add_argument("--port", help="server listen port", default=50051, type=int) 44 | parser.add_argument("--title", help="terminal title", default="OTA status monitor") 45 | 46 | args = parser.parse_args() 47 | main(args.title, args.host, args.port) 48 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_configs/test_cfg_configurable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import pytest 19 | from pytest import MonkeyPatch 20 | 21 | from otaclient.configs import ENV_PREFIX, set_configs 22 | 23 | 24 | @pytest.mark.parametrize( 25 | "setting, env_value, value", 26 | ( 27 | ("DEFAULT_LOG_LEVEL", "CRITICAL", "CRITICAL"), 28 | ( 29 | "LOG_LEVEL_TABLE", 30 | r"""{"ota_metadata": "DEBUG"}""", 31 | {"ota_metadata": "DEBUG"}, 32 | ), 33 | ("DOWNLOAD_INACTIVE_TIMEOUT", "200", 200), 34 | ), 35 | ) 36 | def test_load_configs(setting, env_value, value, monkeypatch: MonkeyPatch): 37 | monkeypatch.setenv(f"{ENV_PREFIX}{setting}", env_value, value) 38 | 39 | mocked_configs = set_configs() 40 | assert getattr(mocked_configs, setting) == value 41 | 42 | 43 | def test_load_default_on_invalid_envs(monkeypatch: MonkeyPatch) -> None: 44 | # first get a normal configs without any envs 45 | normal_cfg = set_configs() 46 | # patch envs 47 | monkeypatch.setenv(f"{ENV_PREFIX}DOWNLOAD_INACTIVE_TIMEOUT", "not_an_int") 48 | # check if config is the defualt one 49 | assert normal_cfg == set_configs() 50 | -------------------------------------------------------------------------------- /proto/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = "hatchling.build" 3 | requires = [ 4 | "grpcio-tools<1.58,>=1.57", 5 | "hatch-vcs", 6 | "hatchling>=1.20", 7 | ] 8 | 9 | [project] 10 | name = "otaclient-pb2" 11 | version = "0.8.0" 12 | readme = "README.md" 13 | requires-python = ">=3.8" 14 | classifiers = [ 15 | "License :: OSI Approved :: Apache Software License", 16 | "Operating System :: Unix", 17 | "Programming Language :: Python :: 3 :: Only", 18 | "Programming Language :: Python :: 3.8", 19 | "Programming Language :: Python :: 3.9", 20 | "Programming Language :: Python :: 3.10", 21 | "Programming Language :: Python :: 3.11", 22 | "Programming Language :: Python :: 3.12", 23 | ] 24 | urls.Source = "https://github.com/tier4/ota-client/proto" 25 | 26 | [tool.hatch.version] 27 | source = "vcs" 28 | 29 | [tool.hatch.metadata] 30 | allow-direct-references = true 31 | 32 | [tool.hatch.build.hooks.vcs] 33 | version-file = "src/otaclient_pb2/_version.py" 34 | 35 | [tool.hatch.build.targets.sdist] 36 | exclude = [ 37 | "whl", 38 | ] 39 | 40 | [tool.hatch.build.targets.wheel] 41 | exclude = [ 42 | "**/.gitignore", 43 | "**/*README.md", 44 | "whl", 45 | ] 46 | only-include = [ 47 | "src", 48 | ] 49 | sources = [ 50 | "src", 51 | ] 52 | 53 | [tool.hatch.build.hooks.custom] 54 | proto_builds = [ 55 | { proto_file = "otaclient_v2.proto", output_package = "otaclient_pb2/v2", api_version = "2.2.0" }, 56 | ] 57 | 58 | [tool.black] 59 | line-length = 88 60 | target-version = [ 61 | 'py38', 62 | ] 63 | extend-exclude = '''( 64 | ^.*(_pb2.pyi?|_pb2_grpc.pyi?)$ 65 | )''' 66 | 67 | [tool.isort] 68 | profile = "black" 69 | extend_skip_glob = [ 70 | "*_pb2.py*", 71 | "_pb2_grpc.py*", 72 | ] 73 | 74 | [tool.pyright] 75 | exclude = [ 76 | "**/__pycache__", 77 | "**/.venv", 78 | ] 79 | ignore = [ 80 | "**/*_pb2.py*", 81 | "**/*_pb2_grpc.py*", 82 | ] 83 | pythonVersion = "3.8" 84 | -------------------------------------------------------------------------------- /.github/workflows/build_test_base_images.yaml: -------------------------------------------------------------------------------- 1 | name: build and push test base images CI 2 | 3 | permissions: 4 | contents: read 5 | packages: write 6 | 7 | on: 8 | push: 9 | branches: 10 | - main 11 | paths: 12 | - "docker/test_base/Dockerfile" 13 | - "docker/test_base/entry_point.sh" 14 | - ".github/workflows/build_test_base_images.yaml" 15 | workflow_dispatch: 16 | 17 | jobs: 18 | build-and-push: 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 60 21 | permissions: 22 | contents: read 23 | packages: write 24 | strategy: 25 | matrix: 26 | ubuntu_version: 27 | - "20.04" 28 | - "22.04" 29 | - "24.04" 30 | steps: 31 | - name: Checkout commit 32 | uses: actions/checkout@v6 33 | 34 | - name: Set up Docker Buildx 35 | uses: docker/setup-buildx-action@v3 36 | with: 37 | # Use buildkit master for zstd compression support 38 | driver-opts: image=moby/buildkit:master 39 | 40 | - name: Log in to GitHub Container Registry 41 | uses: docker/login-action@v3 42 | with: 43 | registry: ghcr.io 44 | username: ${{ github.actor }} 45 | password: ${{ secrets.GITHUB_TOKEN }} 46 | 47 | - name: Build and push 48 | uses: docker/build-push-action@v6 49 | with: 50 | context: ./docker/test_base 51 | file: ./docker/test_base/Dockerfile 52 | push: true 53 | tags: ghcr.io/tier4/ota-client/test_base:ubuntu_${{ matrix.ubuntu_version }} 54 | build-args: | 55 | UBUNTU_BASE=ubuntu:${{ matrix.ubuntu_version }} 56 | outputs: type=image,name=ghcr.io/tier4/ota-client/test_base:ubuntu_${{ matrix.ubuntu_version }},compression=zstd,compression-level=19,oci-mediatypes=true,force-compression=true 57 | cache-from: type=gha 58 | cache-to: type=gha,mode=max 59 | -------------------------------------------------------------------------------- /tests/data/extlinux.conf-r35.4.1-template1: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT primary 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 11 | APPEND ${cbootargs} rw rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 12 | 13 | LABEL with rootfs specified by dev path 14 | MENU LABEL primary kernel 15 | LINUX /boot/Image 16 | INITRD /boot/initrd 17 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 18 | APPEND ${cbootargs} rw root=/dev/mmcblk0p1 rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 19 | 20 | LABEL with rootfs specified by PARTUUID 21 | MENU LABEL primary kernel 22 | LINUX /boot/Image 23 | INITRD /boot/initrd 24 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 25 | APPEND ${cbootargs} rw rootwait root=PARTUUID=aabbccdd rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 26 | 27 | 28 | # When testing a custom kernel, it is recommended that you create a backup of 29 | # the original kernel and add a new entry to this file so that the device can 30 | # fallback to the original kernel. To do this: 31 | # 32 | # 1, Make a backup of the original kernel 33 | # sudo cp /boot/Image /boot/Image.backup 34 | # 35 | # 2, Copy your custom kernel into /boot/Image 36 | # 37 | # 3, Uncomment below menu setting lines for the original kernel 38 | # 39 | # 4, Reboot 40 | 41 | # LABEL backup 42 | # MENU LABEL backup kernel 43 | # LINUX /boot/Image.backup 44 | # INITRD /boot/initrd 45 | # APPEND ${cbootargs} 46 | -------------------------------------------------------------------------------- /tests/data/extlinux.conf-r35.4.1-updated1: -------------------------------------------------------------------------------- 1 | TIMEOUT 30 2 | DEFAULT primary 3 | 4 | MENU TITLE L4T boot options 5 | 6 | LABEL primary 7 | MENU LABEL primary kernel 8 | LINUX /boot/Image 9 | INITRD /boot/initrd 10 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 11 | APPEND ${cbootargs} rw rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb root=PARTUUID=11aa-bbcc-22dd 12 | 13 | LABEL with rootfs specified by dev path 14 | MENU LABEL primary kernel 15 | LINUX /boot/Image 16 | INITRD /boot/initrd 17 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 18 | APPEND ${cbootargs} rw root=PARTUUID=11aa-bbcc-22dd rootwait rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 19 | 20 | LABEL with rootfs specified by PARTUUID 21 | MENU LABEL primary kernel 22 | LINUX /boot/Image 23 | INITRD /boot/initrd 24 | FDT /boot/tegra234-orin-agx-cti-AGX201.dtb 25 | APPEND ${cbootargs} rw rootwait root=PARTUUID=11aa-bbcc-22dd rootfstype=ext4 mminit_loglevel=4 console=ttyTCU0,115200 console=ttyAMA0,115200 console=tty0 firmware_class.path=/etc/firmware fbcon=map:0 net.ifnames=0 nospectre_bhb 26 | 27 | 28 | # When testing a custom kernel, it is recommended that you create a backup of 29 | # the original kernel and add a new entry to this file so that the device can 30 | # fallback to the original kernel. To do this: 31 | # 32 | # 1, Make a backup of the original kernel 33 | # sudo cp /boot/Image /boot/Image.backup 34 | # 35 | # 2, Copy your custom kernel into /boot/Image 36 | # 37 | # 3, Uncomment below menu setting lines for the original kernel 38 | # 39 | # 4, Reboot 40 | 41 | # LABEL backup 42 | # MENU LABEL backup kernel 43 | # LINUX /boot/Image.backup 44 | # INITRD /boot/initrd 45 | # APPEND ${cbootargs} 46 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_log_setting.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | 20 | import pytest 21 | 22 | from otaclient import _logging 23 | 24 | MODULE = _logging.__name__ 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | @pytest.mark.parametrize( 29 | "test_log_msg, test_extra, expected_log_type", 30 | [ 31 | ("emit one logging entry", None, _logging.LogType.LOG), 32 | ( 33 | "emit one logging entry", 34 | {"log_type": _logging.LogType.LOG}, 35 | _logging.LogType.LOG, 36 | ), 37 | ( 38 | "emit one metrics entry", 39 | {"log_type": _logging.LogType.METRICS}, 40 | _logging.LogType.METRICS, 41 | ), 42 | ], 43 | ) 44 | def test_server_logger(test_log_msg, test_extra, expected_log_type): 45 | # ------ setup test ------ # 46 | _handler = _logging._LogTeeHandler() 47 | logger.addHandler(_handler) 48 | 49 | # ------ execution ------ # 50 | logger.info(test_log_msg, extra=test_extra) 51 | 52 | # ------ clenaup ------ # 53 | logger.removeHandler(_handler) 54 | 55 | # ------ check result ------ # 56 | _queue = _handler._queue 57 | _log = _queue.get_nowait() 58 | assert _log is not None 59 | assert _log.log_type == expected_log_type 60 | assert _log.message == test_log_msg 61 | -------------------------------------------------------------------------------- /.github/workflows/release_api.yml: -------------------------------------------------------------------------------- 1 | name: Release API(Protocol Buffer) CI 2 | 3 | permissions: 4 | contents: read 5 | packages: read 6 | actions: read 7 | 8 | on: 9 | release: 10 | types: [published] 11 | # allow manually triggering the build 12 | workflow_dispatch: 13 | 14 | jobs: 15 | build_python_packages: 16 | runs-on: ubuntu-latest 17 | timeout-minutes: 10 18 | steps: 19 | - name: checkout 20 | uses: actions/checkout@v6 21 | 22 | - name: setup python 23 | uses: actions/setup-python@v6 24 | with: 25 | # use the minimum py ver we support to generate the wheel 26 | python-version: 3.8 27 | 28 | - name: install build deps 29 | run: | 30 | python -m pip install -U pip 31 | python -m pip install -U hatch 32 | 33 | - name: build otaclient service API python package 34 | run: | 35 | pushd proto 36 | hatch build -t wheel 37 | popd 38 | mkdir -p dist 39 | cp proto/dist/*.whl dist 40 | 41 | - name: upload artifacts 42 | uses: actions/upload-artifact@v6 43 | with: 44 | name: artifacts-python-packages 45 | path: dist/*.whl 46 | 47 | post_build: 48 | runs-on: ubuntu-latest 49 | needs: [build_python_packages] 50 | permissions: 51 | contents: write 52 | steps: 53 | - name: checkout 54 | uses: actions/checkout@v6 55 | 56 | - name: download python packages artifact 57 | uses: actions/download-artifact@v7 58 | with: 59 | name: artifacts-python-packages 60 | path: dist 61 | 62 | - name: calculate checksum 63 | uses: ./.github/actions/calculate_checksum 64 | with: 65 | file_patterns: '*.{whl}' 66 | directory: 'dist' 67 | 68 | - name: publish release artifacts 69 | if: ${{ github.event_name == 'release' }} 70 | uses: softprops/action-gh-release@v2 71 | with: 72 | files: | 73 | dist/*.whl 74 | dist/*.checksum 75 | -------------------------------------------------------------------------------- /tests/test_ota_proxy/test_subprocess_launch_otaproxy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import multiprocessing as mp 19 | import time 20 | from pathlib import Path 21 | 22 | from ota_proxy import run_otaproxy 23 | 24 | 25 | def otaproxy_process(cache_dir: str): 26 | ota_cache_dir = Path(cache_dir) 27 | ota_cache_db = ota_cache_dir / "cache_db" 28 | 29 | run_otaproxy( 30 | host="127.0.0.1", 31 | port=8082, 32 | init_cache=True, 33 | cache_dir=str(ota_cache_dir), 34 | cache_db_f=str(ota_cache_db), 35 | upper_proxy="", 36 | enable_cache=True, 37 | enable_https=False, 38 | ) 39 | 40 | 41 | def test_subprocess_start_otaproxy(tmp_path: Path): 42 | # --- setup --- # 43 | (ota_cache_dir := tmp_path / "ota-cache").mkdir(exist_ok=True) 44 | ota_cache_db = ota_cache_dir / "cache_db" 45 | 46 | # --- execution --- # 47 | _spawn_ctx = mp.get_context("spawn") 48 | 49 | otaproxy_subprocess = _spawn_ctx.Process( 50 | target=otaproxy_process, args=(ota_cache_dir,) 51 | ) 52 | otaproxy_subprocess.start() 53 | time.sleep(3) # wait for subprocess to finish up initializing 54 | 55 | # --- assertion --- # 56 | try: 57 | assert otaproxy_subprocess.is_alive() 58 | assert ota_cache_db.is_file() 59 | finally: 60 | otaproxy_subprocess.terminate() 61 | otaproxy_subprocess.join() 62 | -------------------------------------------------------------------------------- /tests/keys/gen_certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # https://stackoverflow.com/questions/52500165/problem-verifying-a-self-created-openssl-root-intermediate-and-end-user-certifi 4 | 5 | set -eux 6 | 7 | CA_CHAIN_PREFIX=${1:-test} 8 | 9 | # Root CA: 10 | openssl ecparam -out root.key -name prime256v1 -genkey 11 | openssl req -new -x509 \ 12 | -days $((365 * 100)) \ 13 | -key root.key \ 14 | -out ${CA_CHAIN_PREFIX}.root.pem \ 15 | -sha256 \ 16 | -subj "/C=JP/ST=Tokyo/O=Tier4/CN=root.tier4.jp" 17 | 18 | # Intermediate 19 | openssl ecparam -out interm.key -name prime256v1 -genkey 20 | openssl req -new \ 21 | -key interm.key \ 22 | -out interm.csr \ 23 | -sha256 \ 24 | -subj "/C=JP/ST=Tokyo/O=Tier4/CN=intermediate.tier4.jp" 25 | 26 | CA_INTERM_EXT=" 27 | [ v3_intermediate_ca ] 28 | subjectKeyIdentifier = hash 29 | authorityKeyIdentifier = keyid:always,issuer 30 | basicConstraints = critical, CA:true, pathlen:0 31 | " 32 | openssl x509 -req \ 33 | -days $((365 * 100)) \ 34 | -in interm.csr \ 35 | -CA ${CA_CHAIN_PREFIX}.root.pem \ 36 | -CAkey root.key \ 37 | -out ${CA_CHAIN_PREFIX}.interm.pem \ 38 | -sha256 -CAcreateserial \ 39 | -extfile <(echo "${CA_INTERM_EXT}") \ 40 | -extensions v3_intermediate_ca 41 | 42 | # Sign cert 43 | SIGN_CERT_EXT=" 44 | [ sign_cert ] 45 | subjectKeyIdentifier = hash 46 | authorityKeyIdentifier = keyid:always,issuer 47 | keyUsage = critical, digitalSignature 48 | extendedKeyUsage = codeSigning 49 | basicConstraints = critical, CA:FALSE 50 | " 51 | 52 | openssl ecparam -out sign.key -name prime256v1 -genkey 53 | openssl req -new \ 54 | -key sign.key \ 55 | -out sign.csr \ 56 | -sha256 \ 57 | -subj "/C=JP/ST=Tokyo/O=Tier4/CN=sign.tier4.jp" 58 | 59 | openssl x509 -req \ 60 | -days $((365 * 100)) \ 61 | -in sign.csr \ 62 | -CA ${CA_CHAIN_PREFIX}.interm.pem \ 63 | -CAkey interm.key \ 64 | -out sign.pem \ 65 | -sha256 -CAcreateserial \ 66 | -extfile <(echo "${SIGN_CERT_EXT}") \ 67 | -extensions sign_cert 68 | 69 | rm -f root.key interm.key interm.csr sign.csr *.srl 70 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test CI 2 | 3 | permissions: 4 | contents: read 5 | packages: read 6 | actions: read 7 | 8 | on: 9 | pull_request: 10 | branches: 11 | - main 12 | - v* 13 | types: 14 | - opened 15 | - synchronize 16 | - reopened 17 | # NOTE(20241016): this is a workaround for PR with head 18 | # updated by gen_requirements_txt workflow 19 | - review_requested 20 | - assigned 21 | push: 22 | branches: 23 | - main 24 | - v* 25 | paths: 26 | - "src/**" 27 | - "tests/**" 28 | - ".github/workflows/test.yaml" 29 | # allow the test CI to be manually triggerred 30 | workflow_dispatch: 31 | 32 | jobs: 33 | pytest_directly_on_supported_os: 34 | runs-on: ubuntu-latest 35 | timeout-minutes: 20 36 | strategy: 37 | fail-fast: false 38 | matrix: 39 | test_base: 40 | - ubuntu-20.04 41 | - ubuntu-22.04 42 | - ubuntu-24.04 43 | steps: 44 | - name: Checkout commit 45 | uses: actions/checkout@v6 46 | - name: Execute pytest with coverage trace under ota-test_base container 47 | run: | 48 | mkdir -p test_result 49 | docker compose -f docker/test_base/docker-compose_tests.yml run --rm tester-${{ matrix.test_base }} 50 | 51 | # NOTE: we only intend to test otaclient with python3.13 on 52 | # ubuntu 22.04, as this is the setup for the otaclient app image build. 53 | pytest_with_py313_on_ubuntu_2204: 54 | runs-on: ubuntu-latest 55 | timeout-minutes: 20 56 | steps: 57 | - name: Checkout commit 58 | uses: actions/checkout@v6 59 | # sonarcloud needs full git histories 60 | with: 61 | fetch-depth: 0 62 | - name: Execute pytest with coverage trace under ota-py313-app container 63 | run: | 64 | mkdir -p test_result 65 | docker compose -f docker/test_base/docker-compose_tests_py313.yml run --rm tester-ubuntu-22.04 66 | - name: SonarCloud Scan 67 | uses: SonarSource/sonarqube-scan-action@v7.0.0 68 | continue-on-error: true 69 | env: 70 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 71 | -------------------------------------------------------------------------------- /src/otaclient/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """otaclient configs package.""" 15 | 16 | from typing import TYPE_CHECKING 17 | 18 | from otaclient.configs._cfg_configurable import ( 19 | ENV_PREFIX, 20 | ConfigurableSettings, 21 | set_configs, 22 | ) 23 | from otaclient.configs._cfg_consts import Consts, CreateStandbyMechanism, dynamic_root 24 | from otaclient.configs._ecu_info import BootloaderType, ECUContact, ECUInfo 25 | from otaclient.configs._proxy_info import ProxyInfo 26 | 27 | __all__ = [ 28 | "ENV_PREFIX", 29 | "ConfigurableSettings", 30 | "Consts", 31 | "CreateStandbyMechanism", 32 | "BootloaderType", 33 | "ECUContact", 34 | "ECUInfo", 35 | "ProxyInfo", 36 | "set_configs", 37 | "dynamic_root", 38 | "DefaultOTAClientConfigs", 39 | ] 40 | 41 | if TYPE_CHECKING: 42 | 43 | class DefaultOTAClientConfigs(ConfigurableSettings, Consts): 44 | """Default OTAClient configs, without parsing the env vars.""" 45 | 46 | else: 47 | 48 | class DefaultOTAClientConfigs: 49 | 50 | def __init__(self) -> None: 51 | self._cfg_consts = Consts() 52 | self._cfg_configurable = ConfigurableSettings() 53 | 54 | # NOTE(20241108): still use __getattr__ to allow changing/mocking attributes 55 | # for easy testing. 56 | def __getattr__(self, name: str): 57 | for _cfg in [self._cfg_consts, self._cfg_configurable]: 58 | try: 59 | return getattr(_cfg, name) 60 | except AttributeError: 61 | continue 62 | raise AttributeError(f"no such config field: {name=}") 63 | -------------------------------------------------------------------------------- /src/otaclient_api/v2/api_stub.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from typing import Any 19 | 20 | from otaclient_api.v2 import _types 21 | from otaclient_api.v2 import otaclient_v2_pb2 as pb2 22 | from otaclient_api.v2 import otaclient_v2_pb2_grpc as pb2_grpc 23 | 24 | 25 | class OtaClientServiceV2(pb2_grpc.OtaClientServiceServicer): 26 | def __init__(self, ota_client_stub: Any): 27 | self._stub = ota_client_stub 28 | 29 | async def Update(self, request: pb2.UpdateRequest, context) -> pb2.UpdateResponse: 30 | response = await self._stub.update(_types.UpdateRequest.convert(request)) 31 | return response.export_pb() 32 | 33 | async def Stop(self, request: pb2.StopRequest, context) -> pb2.StopResponse: 34 | response = await self._stub.stop(_types.StopRequest.convert(request)) 35 | return response.export_pb() 36 | 37 | async def Rollback( 38 | self, request: pb2.RollbackRequest, context 39 | ) -> pb2.RollbackResponse: 40 | response = await self._stub.rollback(_types.RollbackRequest.convert(request)) 41 | return response.export_pb() 42 | 43 | async def ClientUpdate( 44 | self, request: pb2.UpdateRequest, context 45 | ) -> pb2.UpdateResponse: 46 | response = await self._stub.client_update( 47 | _types.ClientUpdateRequest.convert(request) 48 | ) 49 | return response.export_pb() 50 | 51 | async def Status(self, request: pb2.StatusRequest, context) -> pb2.StatusResponse: 52 | response = await self._stub.status(_types.StatusRequest.convert(request)) 53 | return response.export_pb() 54 | -------------------------------------------------------------------------------- /docker/app_img/Dockerfile: -------------------------------------------------------------------------------- 1 | # NOTE: due to newer version of mkfs.ext4 will enable some flags 2 | # by default that are not supported by older version of ubuntu, 3 | # we have to pin the ubuntu base to 22.04. 4 | # NOTE: due to glibc compat issue, we need to pair ubuntu:22.04 with bullseye. 5 | ARG UBUNTU_BASE=ubuntu:22.04 6 | ARG BUILD_BASE=python:3.13-bullseye 7 | 8 | ARG UV_VERSION=0.9.4 9 | 10 | FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv 11 | 12 | FROM ${BUILD_BASE} AS app_build 13 | 14 | ARG PYINSTALLER_VER=6.16.0 15 | 16 | RUN mkdir /build 17 | # minimum required files for building app 18 | COPY ./src /build/src 19 | COPY ./scripts /build/scripts 20 | COPY ./pyproject.toml ./uv.lock ./README.md /build/ 21 | 22 | COPY --from=uv /uv /uvx /bin/ 23 | 24 | WORKDIR /build 25 | # NOTE: mount .git for hatch-vcs to determine the version 26 | RUN --mount=type=bind,source=./.git,target=/build/.git,ro \ 27 | set -eux; \ 28 | apt-get update -qq; \ 29 | apt-get install -y -qq --no-install-recommends \ 30 | git python3 python3-setuptools; \ 31 | uv run --with=pyinstaller==${PYINSTALLER_VER} --locked --no-managed-python --python=3.13 \ 32 | pyinstaller -D -s --name otaclient src/otaclient/__main__.py 33 | 34 | FROM ${UBUNTU_BASE} 35 | 36 | ARG OTACLIENT_INSTALLATION="/otaclient" 37 | ARG COMPAT_VENV_PATH="/otaclient/venv/bin" 38 | 39 | COPY --from=app_build /build/dist/otaclient ${OTACLIENT_INSTALLATION} 40 | 41 | RUN set -eux; \ 42 | # add necessary runtime directories for OTA 43 | mkdir -p /mnt /ota-cache /host_root /usr/share/ca-certificates /usr/share/zoneinfo; \ 44 | # add compatible layer for otaclient launching the dynamic otaclient app, 45 | # see otaclient.main module for more details. 46 | mkdir -p ${COMPAT_VENV_PATH} ;\ 47 | echo '#!/bin/bash\n'${OTACLIENT_INSTALLATION}'/otaclient\n' > ${COMPAT_VENV_PATH}/python3; \ 48 | chmod +x ${COMPAT_VENV_PATH}/python3; \ 49 | apt-get update -qq; \ 50 | apt-get install --no-install-recommends -y \ 51 | # for e2label, mkfs.ext4 52 | e2fsprogs \ 53 | # for TLS certs verification 54 | ca-certificates; \ 55 | apt-get clean; \ 56 | rm -rf \ 57 | /root/.cache \ 58 | /tmp/* \ 59 | /var/lib/apt/lists/* \ 60 | /var/tmp/* 61 | -------------------------------------------------------------------------------- /src/otaclient_common/_env.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import functools 16 | import os 17 | import sys 18 | from typing import Optional 19 | 20 | from otaclient.configs.cfg import cfg 21 | from otaclient_common import replace_root 22 | 23 | try: 24 | cache = functools.cache # type: ignore[attr-defined] 25 | except AttributeError: 26 | cache = functools.lru_cache(maxsize=None) 27 | 28 | RUN_AS_PYINSTALLER_BUNDLE = getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS") 29 | 30 | 31 | @cache 32 | def is_running_as_app_image() -> bool: 33 | """Check if is the systemd managed client running.""" 34 | return bool(os.getenv(cfg.RUNNING_AS_APP_IMAGE)) 35 | 36 | 37 | @cache 38 | def is_running_as_downloaded_dynamic_app() -> bool: 39 | """Check if is the downloaded dynamic client running.""" 40 | return bool(os.getenv(cfg.RUNNING_DOWNLOADED_DYNAMIC_OTA_CLIENT)) 41 | 42 | 43 | @cache 44 | def is_dynamic_client_running() -> bool: 45 | """Check if the dynamic client is running.""" 46 | return is_running_as_app_image() or is_running_as_downloaded_dynamic_app() 47 | 48 | 49 | @cache 50 | def get_dynamic_client_chroot_path() -> Optional[str]: 51 | """Get the chroot path.""" 52 | if is_dynamic_client_running(): 53 | return cfg.DYNAMIC_CLIENT_MNT_HOST_ROOT 54 | return None 55 | 56 | 57 | @cache 58 | def get_otaclient_squashfs_download_dst() -> str: 59 | """Get the location to hold downloaded otaclient squashfs image.""" 60 | if is_dynamic_client_running(): 61 | return replace_root( 62 | cfg.DYNAMIC_CLIENT_SQUASHFS_FILE, 63 | cfg.CANONICAL_ROOT, 64 | cfg.DYNAMIC_CLIENT_MNT_HOST_ROOT, 65 | ) 66 | return cfg.DYNAMIC_CLIENT_SQUASHFS_FILE 67 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_logging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import time 17 | 18 | from pytest import LogCaptureFixture 19 | 20 | from otaclient_common import _logging as _logging 21 | 22 | TEST_ROUND = 10 23 | TEST_LOGGINGS_NUM = 3000 24 | 25 | 26 | def test_burst_logging(caplog: LogCaptureFixture): 27 | logger_name = "upper_logger.intermediate_logger.this_logger" 28 | upper_logger = "upper_logger.intermediate_logger" 29 | 30 | burst_round_length = 1 31 | 32 | logger = _logging.get_burst_suppressed_logger( 33 | logger_name, 34 | # NOTE: test upper_logger_name calculated from logger_name 35 | burst_max=1, 36 | burst_round_length=burst_round_length, 37 | ) 38 | 39 | # test loggging suppressing 40 | # NOTE: outer loop ensures that suppression only works 41 | # within each burst_round, and should be refresed 42 | # in new round. 43 | for _ in range(TEST_ROUND): 44 | for idx in range(TEST_LOGGINGS_NUM): 45 | logger.error(idx) 46 | time.sleep(burst_round_length * 2) 47 | logger.error("burst_round end") 48 | 49 | # For each round, the loggings will be as follow: 50 | # 1. logger.error(idx) # idx==0 51 | # 2. a warning of exceeded loggings are suppressed 52 | # 3. a warning of how many loggings are suppressed 53 | # 4. logger.error("burst_round end") 54 | # NOTE that the logger.error("burst_round end") will be included in the 55 | # next burst_suppressing roud, so excepts for the first round, we will 56 | # only have three records. 57 | assert len(records := caplog.records) <= 4 58 | # warning msg comes from upper_logger 59 | assert records[1].name == upper_logger 60 | caplog.clear() 61 | -------------------------------------------------------------------------------- /tools/local_ota/api_stop_v2.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """A simple tool to stop an OTA locally, for API version 2.""" 15 | 16 | 17 | from __future__ import annotations 18 | 19 | import argparse 20 | import asyncio 21 | import logging 22 | 23 | from otaclient_api.v2 import _types 24 | from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | DEFAULT_PORT = 50051 29 | 30 | 31 | async def main( 32 | host: str, 33 | port: int, 34 | *, 35 | req: _types.StopRequest, 36 | timeout: int = 3, 37 | ) -> None: 38 | try: 39 | resp = await OTAClientCall.stop_call( 40 | "not_used", 41 | host, 42 | port, 43 | request=req, 44 | timeout=timeout, 45 | ) 46 | logger.info(f"stop response: {resp}") 47 | except ECUNoResponse as e: 48 | _err_msg = f"ECU doesn't response to the request on-time({timeout=}): {e}" 49 | logger.error(_err_msg) 50 | 51 | 52 | if __name__ == "__main__": 53 | logging.basicConfig(level=logging.INFO) 54 | 55 | parser = argparse.ArgumentParser( 56 | description="Calling ECU's stop API, API version v2", 57 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 58 | ) 59 | 60 | parser.add_argument( 61 | "-i", 62 | "--host", 63 | help="ECU listen IP address", 64 | required=True, 65 | ) 66 | parser.add_argument( 67 | "-p", 68 | "--port", 69 | help="ECU listen port", 70 | type=int, 71 | default=DEFAULT_PORT, 72 | ) 73 | 74 | args = parser.parse_args() 75 | 76 | stop_request = _types.StopRequest() 77 | asyncio.run( 78 | main( 79 | args.host, 80 | args.port, 81 | req=stop_request, 82 | ) 83 | ) 84 | -------------------------------------------------------------------------------- /.github/workflows/sync_lockfile.yaml: -------------------------------------------------------------------------------- 1 | name: Sync uv.lock on pyproject.toml and exporting requirements.txt. 2 | 3 | permissions: 4 | contents: read 5 | packages: read 6 | actions: read 7 | 8 | on: 9 | pull_request: 10 | branches: 11 | - main 12 | - v.* 13 | paths: 14 | - 'uv.lock' 15 | - 'pyproject.toml' 16 | - .github/workflows/sync_lockfile.yaml 17 | # allow manual dispatch of this workflow 18 | workflow_dispatch: 19 | 20 | jobs: 21 | sync-lockfile: 22 | runs-on: ubuntu-latest 23 | permissions: 24 | contents: write 25 | timeout-minutes: 1 26 | steps: 27 | - name: Checkout repository 28 | uses: actions/checkout@v6 29 | with: 30 | ref: ${{ github.event.pull_request.head.ref }} 31 | # use repo scope deploy key for the later git operations, so that the pushed commit can trigger the 32 | # workflow as expected. The default action token GITHUB_TOKEN cannot trigger new workflows. 33 | # For more details about this restriction, please refer to: 34 | # https://github.com/peter-evans/create-pull-request/issues/48 and 35 | # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs 36 | ssh-key: ${{ secrets.DEPLOY_KEY }} 37 | persist-credentials: true 38 | - name: Install uv 39 | uses: astral-sh/setup-uv@v7 40 | with: 41 | version: "0.8.24" 42 | - name: Setup python 43 | uses: actions/setup-python@v6 44 | with: 45 | python-version-file: ".python-version" 46 | - name: Sync uv.lock with pyproject.toml 47 | run: uv lock 48 | - name: Export requirements.txt from uv.lock 49 | run: uv export --frozen --no-dev --no-editable --no-hashes --no-emit-project > requirements.txt 50 | - name: Commit change if needed 51 | run: | 52 | git config --global user.name "github-actions[bot]+sync-lockfile" 53 | git config --global user.email "github-actions[bot]+sync-lockfile@users.noreply.github.com" 54 | 55 | if git diff --exit-code uv.lock requirements.txt; then 56 | echo "Skip commit as no changes are made" 57 | else 58 | echo "Commit changes on update ..." 59 | git add uv.lock requirements.txt 60 | git commit -m "[GHA] Update uv.lock and/or requirements.txt" 61 | git push 62 | fi 63 | -------------------------------------------------------------------------------- /scripts/hatch_build_lock_deps.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # A custom metadata hook implementation that overwrites dependencies with locked version 16 | # from uv.lock file. 17 | # 18 | # reference https://github.com/pga2rn/py_deps_locked_build_demo 19 | 20 | from __future__ import annotations 21 | 22 | import os 23 | import subprocess 24 | from pprint import pformat 25 | 26 | from hatchling.metadata.plugin.interface import MetadataHookInterface 27 | 28 | # NOTE: ! VERY IMPORTANT ! 29 | # We should ONLY enable dependencies overwrite when needed, otherwise 30 | # `uv sync` or install project as editable will not work due to conflicts! 31 | HINT_ENV_NAME = "ENABLE_DEPS_LOCKED_BUILD" 32 | HINT_ENV_VALUE = "yes" 33 | 34 | # fmt: off 35 | UV_EXPORT_FREEZED_CMD = ( 36 | "uv", "export", "--frozen", "--color=never", 37 | "--no-dev", "--no-editable", "--no-hashes", "--no-emit-project", 38 | ) 39 | # fmt: on 40 | 41 | 42 | def _uv_export_locked_requirements() -> list[str]: 43 | print( 44 | f"metahook: generate locked deps list with: {' '.join(UV_EXPORT_FREEZED_CMD)}" 45 | ) 46 | _uv_export_res = subprocess.run(UV_EXPORT_FREEZED_CMD, capture_output=True) 47 | _res: list[str] = [] 48 | for _dep_line in _uv_export_res.stdout.decode().splitlines(): 49 | _dep_line = _dep_line.strip() 50 | if _dep_line.startswith("#"): 51 | continue 52 | _res.append(_dep_line) 53 | return _res 54 | 55 | 56 | class CustomMetadataHook(MetadataHookInterface): 57 | def update(self, metadata) -> None: 58 | if os.environ.get(HINT_ENV_NAME) == HINT_ENV_VALUE: 59 | print(f"metahook: {HINT_ENV_NAME} is configured.") 60 | print("metahook: will overwrite dependencies with locked version.") 61 | _locked_deps = _uv_export_locked_requirements() 62 | print(f"metahook: locked deps: {pformat(_locked_deps)}") 63 | metadata["dependencies"] = _locked_deps 64 | -------------------------------------------------------------------------------- /src/ota_proxy/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | from hashlib import sha256 5 | from os import PathLike 6 | from typing import AsyncGenerator 7 | from urllib.parse import SplitResult, quote, urlsplit 8 | 9 | import anyio 10 | from anyio import open_file 11 | 12 | from otaclient_common._typing import StrOrPath 13 | 14 | from .config import config as cfg 15 | 16 | 17 | async def read_file( 18 | fpath: PathLike, chunk_size: int = cfg.LOCAL_READ_SIZE 19 | ) -> AsyncGenerator[bytes]: 20 | """Open and read a file asynchronously.""" 21 | async with await open_file(fpath, "rb") as f: 22 | fd = f.wrapped.fileno() 23 | os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL) 24 | while data := await f.read(chunk_size): 25 | yield data 26 | os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) 27 | 28 | 29 | def read_file_once(fpath: StrOrPath | anyio.Path) -> bytes: 30 | with open(fpath, "rb") as f: 31 | fd = f.fileno() 32 | os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL) 33 | data = f.read() 34 | os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) 35 | return data 36 | 37 | 38 | def url_based_hash(raw_url: str) -> str: 39 | """Generate sha256hash with unquoted raw_url.""" 40 | _sha256_value = sha256(raw_url.encode()).hexdigest() 41 | return f"{cfg.URL_BASED_HASH_PREFIX}{_sha256_value}" 42 | 43 | 44 | def process_raw_url(raw_url: str, enable_https: bool) -> str: 45 | """Process the raw URL received from upper uvicorn app. 46 | 47 | NOTE: raw_url(get from uvicorn) is unquoted, we must quote it again before we send it to the remote 48 | NOTE(20221003): as otaproxy, we should treat all contents after netloc as path and not touch it, 49 | because we should forward the request as it to the remote. 50 | NOTE(20221003): unconditionally set scheme to https if enable_https, else unconditionally set to http 51 | """ 52 | _raw_parse = urlsplit(raw_url) 53 | # get the base of the raw_url, which is :// 54 | _raw_base = SplitResult( 55 | scheme=_raw_parse.scheme, 56 | netloc=_raw_parse.netloc, 57 | path="", 58 | query="", 59 | fragment="", 60 | ).geturl() 61 | 62 | # get the leftover part of URL besides base as path, and then quote it 63 | # finally, regenerate proper quoted url 64 | return SplitResult( 65 | scheme="https" if enable_https else "http", 66 | netloc=_raw_parse.netloc, 67 | path=quote(raw_url.replace(_raw_base, "", 1)), 68 | query="", 69 | fragment="", 70 | ).geturl() 71 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | import time 20 | 21 | import pytest 22 | 23 | from otaclient._utils import wait_and_log 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | class _TickingFlag: 29 | 30 | def __init__(self, trigger_in: int) -> None: 31 | self._trigger_time = time.time() + trigger_in 32 | 33 | def is_set(self) -> bool: 34 | _now = time.time() 35 | return _now > self._trigger_time 36 | 37 | 38 | def test_wait_and_log(caplog: pytest.LogCaptureFixture): 39 | # NOTE: allow 2 more seconds for expected_trigger_time 40 | trigger_in, expected_trigger_time = 11, time.time() + 11 + 2 41 | _flag = _TickingFlag(trigger_in=trigger_in) 42 | _msg = "ticking flag" 43 | 44 | result = wait_and_log( 45 | _flag.is_set, 46 | _msg, 47 | check_for=True, 48 | check_interval=1, 49 | log_interval=2, 50 | log_func=logger.warning, 51 | ) 52 | 53 | assert result is True 54 | assert len(caplog.records) == 5 55 | assert caplog.records[0].levelno == logging.WARNING 56 | assert caplog.records[0].msg == f"wait for {_msg}: 2s passed ..." 57 | assert time.time() < expected_trigger_time 58 | 59 | 60 | def test_wait_and_log_timeout(caplog: pytest.LogCaptureFixture): 61 | # Create a flag that will never be set (trigger time is very far in the future) 62 | _flag = _TickingFlag(trigger_in=3600) # 1 hour in the future 63 | _msg = "condition that will timeout" 64 | timeout = 5 # timeout after 5 seconds 65 | 66 | # Test that wait_and_log returns False when it times out 67 | result = wait_and_log( 68 | _flag.is_set, 69 | _msg, 70 | check_for=True, 71 | check_interval=1, 72 | log_interval=2, 73 | log_func=logger.warning, 74 | timeout=timeout, 75 | ) 76 | 77 | # Verify the function returned False (condition wasn't met) 78 | assert result is False 79 | -------------------------------------------------------------------------------- /src/ota_proxy/external_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Implementation of mounting/umounting external cache.""" 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | 20 | from ota_proxy.config import config 21 | from otaclient_common import cmdhelper 22 | from otaclient_common._typing import StrOrPath 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | def mount_external_cache( 28 | mnt_point: StrOrPath, 29 | *, 30 | cache_dev_fslabel: str = config.EXTERNAL_CACHE_DEV_FSLABEL, 31 | ) -> StrOrPath | None: 32 | logger.info( 33 | f"otaproxy will try to detect external cache dev and mount to {mnt_point}" 34 | ) 35 | 36 | _cache_dev = cmdhelper.get_dev_by_token( 37 | "LABEL", 38 | cache_dev_fslabel, 39 | raise_exception=False, 40 | ) 41 | if not _cache_dev: 42 | logger.info("no cache dev is attached") 43 | return 44 | 45 | if len(_cache_dev) > 1: 46 | logger.warning( 47 | f"multiple external cache storage device found, use the first one: {_cache_dev[0]}" 48 | ) 49 | _cache_dev = _cache_dev[0] 50 | logger.info(f"external cache dev detected at {_cache_dev}") 51 | 52 | try: 53 | cmdhelper.ensure_mount_point(mnt_point, ignore_error=True) 54 | cmdhelper.ensure_mount( 55 | target=_cache_dev, 56 | mnt_point=mnt_point, 57 | mount_func=cmdhelper.mount_ro, 58 | raise_exception=True, 59 | max_retry=3, 60 | ) 61 | logger.info( 62 | f"successfully mount external cache dev {_cache_dev} on {mnt_point}" 63 | ) 64 | return mnt_point 65 | except Exception as e: 66 | logger.warning(f"failed to mount external cache: {e!r}") 67 | 68 | 69 | def umount_external_cache(mnt_point: StrOrPath) -> None: 70 | try: 71 | cmdhelper.ensure_umount(mnt_point, ignore_error=False) 72 | except Exception as e: 73 | logger.warning(f"failed to umount external cache {mnt_point=}: {e!r}") 74 | -------------------------------------------------------------------------------- /.github/actions/build_squashfs_image/action.yml: -------------------------------------------------------------------------------- 1 | name: "Build SquashFS Image" 2 | description: "Builds a SquashFS image using Docker" 3 | inputs: 4 | platform: 5 | description: "The platform to build for" 6 | required: true 7 | platform_suffix: 8 | description: "The platform suffix" 9 | required: true 10 | base_image: 11 | description: "The base image to use" 12 | required: true 13 | version: 14 | description: "The OTAClient version" 15 | required: true 16 | output_squashfs: 17 | description: "The output SquashFS file" 18 | required: true 19 | runs: 20 | using: "composite" 21 | steps: 22 | - name: set temporary variables 23 | id: set-vars 24 | env: 25 | PLATFORM_SUFFIX: ${{ inputs.platform_suffix }} 26 | VERSION: ${{ inputs.version }} 27 | shell: bash 28 | run: | 29 | echo "DOCKER_IMAGE=otaclient-${PLATFORM_SUFFIX}:${VERSION}" >> $GITHUB_ENV 30 | echo "DOCKER_CONTAINER=otaclient-${PLATFORM_SUFFIX}_v${VERSION}" >> $GITHUB_ENV 31 | 32 | - name: build docker image 33 | env: 34 | DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} 35 | PLATFORM: ${{ inputs.platform }} 36 | PLATFORM_SUFFIX: ${{ inputs.platform_suffix }} 37 | BASE_IMAGE: ${{ inputs.base_image }} 38 | VERSION: ${{ inputs.version }} 39 | shell: bash 40 | run: | 41 | docker build -f docker/app_img/Dockerfile --platform ${PLATFORM} \ 42 | --build-arg=UBUNTU_BASE=${BASE_IMAGE} \ 43 | -t ${DOCKER_IMAGE} \ 44 | . 45 | 46 | - name: export squashfs image 47 | env: 48 | DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} 49 | DOCKER_CONTAINER: ${{ env.DOCKER_CONTAINER }} 50 | SQUASHFS: ${{ inputs.output_squashfs }} 51 | shell: bash 52 | # although zstd squashfs kernel support is available since ubuntu 18.04, 53 | # unfortunately it is not enabled by default on some of the systems we need to support. 54 | run: | 55 | mkdir -p $(dirname ${SQUASHFS}) 56 | docker create --name ${DOCKER_CONTAINER} ${DOCKER_IMAGE} 57 | docker export ${DOCKER_CONTAINER} | mksquashfs - ${SQUASHFS} \ 58 | -tar -b 1M \ 59 | -mkfs-time 1729810800 \ 60 | -all-time 1729810800 \ 61 | -no-xattrs \ 62 | -all-root \ 63 | -progress \ 64 | -comp gzip 65 | 66 | - name: cleanup 67 | env: 68 | DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} 69 | DOCKER_CONTAINER: ${{ env.DOCKER_CONTAINER }} 70 | shell: bash 71 | run: | 72 | docker rm ${DOCKER_CONTAINER} 73 | docker rmi ${DOCKER_IMAGE} 74 | -------------------------------------------------------------------------------- /src/otaclient/boot_control/protocol.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from abc import abstractmethod 19 | from pathlib import Path 20 | from typing import Protocol 21 | 22 | from otaclient._types import OTAStatus 23 | 24 | 25 | class BootControllerProtocol(Protocol): 26 | """Boot controller protocol for otaclient.""" 27 | 28 | @abstractmethod 29 | def get_booted_ota_status(self) -> OTAStatus: 30 | """Get the ota_status loaded from status file during otaclient starts up. 31 | 32 | This value is meant to be used only once during otaclient starts up, 33 | to init the live_ota_status maintained by otaclient. 34 | """ 35 | 36 | @abstractmethod 37 | def get_standby_slot_path(self) -> Path: 38 | """Get the Path points to the standby slot mount point.""" 39 | 40 | @property 41 | @abstractmethod 42 | def bootloader_type(self) -> str: 43 | """The type of bootloader.""" 44 | 45 | @property 46 | @abstractmethod 47 | def standby_slot_dev(self) -> Path: 48 | """The device of the standby slot.""" 49 | 50 | @abstractmethod 51 | def get_standby_slot_dev(self) -> str: 52 | """Get the dev to the standby slot.""" 53 | 54 | @abstractmethod 55 | def load_version(self) -> str: 56 | """Read the version info from the current slot.""" 57 | 58 | @abstractmethod 59 | def load_standby_slot_version(self) -> str: 60 | """Read the version info from the standby slot.""" 61 | 62 | @abstractmethod 63 | def on_operation_failure(self) -> None: 64 | """Cleanup by boot_control implementation when OTA failed.""" 65 | 66 | # 67 | # ------ update ------ # 68 | # 69 | 70 | @abstractmethod 71 | def pre_update(self, *, standby_as_ref: bool, erase_standby: bool): ... 72 | 73 | @abstractmethod 74 | def post_update(self, update_version: str) -> None: ... 75 | 76 | @abstractmethod 77 | def finalizing_update(self, *, chroot: str | None = None) -> None: 78 | """Normally this method only reboots the device.""" 79 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_ca_store.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import shutil 19 | import subprocess 20 | from pathlib import Path 21 | 22 | import pytest 23 | from cryptography.x509 import load_pem_x509_certificate 24 | 25 | from ota_metadata.utils.cert_store import CACertStoreInvalid, load_ca_cert_chains 26 | from tests.conftest import TEST_DIR 27 | from tests.conftest import TestConfiguration as cfg 28 | 29 | GEN_CERTS_SCRIPT = TEST_DIR / "keys" / "gen_certs.sh" 30 | TEST_BASE_SIGN_PEM = Path(cfg.CERTS_DIR) / "sign.pem" 31 | 32 | 33 | @pytest.fixture 34 | def setup_ca_chain(tmp_path: Path) -> tuple[str, Path, Path]: 35 | """Create the certs dir and generate certs. 36 | 37 | Returns: 38 | A tuple of chain prefix, sign cert path and certs dir. 39 | """ 40 | certs_dir = tmp_path / "certs" 41 | certs_dir.mkdir(parents=True, exist_ok=True) 42 | 43 | shutil.copy(GEN_CERTS_SCRIPT, certs_dir) 44 | gen_certs_script = certs_dir / GEN_CERTS_SCRIPT.name 45 | 46 | chain = "test_chain" 47 | subprocess.run( 48 | [ 49 | "bash", 50 | str(gen_certs_script), 51 | chain, 52 | ], 53 | cwd=certs_dir, 54 | ) 55 | return chain, certs_dir / "sign.pem", certs_dir 56 | 57 | 58 | def test_ca_store(setup_ca_chain: tuple[str, Path, Path]): 59 | _, sign_pem, certs_dir = setup_ca_chain 60 | sign_cert = load_pem_x509_certificate(sign_pem.read_bytes()) 61 | 62 | ca_store = load_ca_cert_chains(certs_dir) 63 | assert ca_store.verify(sign_cert) 64 | 65 | 66 | def test_ca_store_empty(tmp_path: Path): 67 | with pytest.raises(CACertStoreInvalid): 68 | load_ca_cert_chains(tmp_path) 69 | 70 | 71 | def test_ca_store_invalid(tmp_path: Path): 72 | # create invalid certs under tmp_path 73 | root_cert = tmp_path / "test.root.pem" 74 | intermediate = tmp_path / "test.intermediate.pem" 75 | 76 | root_cert.write_text("abcdef") 77 | intermediate.write_text("123456") 78 | 79 | with pytest.raises(CACertStoreInvalid): 80 | load_ca_cert_chains(tmp_path) 81 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_ota_image_v1/test_cert_verification.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | import logging 18 | import os 19 | import shutil 20 | import subprocess 21 | from pathlib import Path 22 | 23 | import pytest 24 | from cryptography.x509 import load_pem_x509_certificate 25 | 26 | from ota_metadata.utils.cert_store import load_ca_store 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | CERT_GEN_SCRIPT = Path(__file__).parent.parent.parent / "keys" / "gen_certs.sh" 31 | CHAINS = ["dev", "stg", "prd"] 32 | 33 | 34 | @pytest.fixture 35 | def gen_ca_chains(tmp_path: Path) -> tuple[Path, Path, Path]: 36 | """ 37 | Check tests/keys/gen_certs.sh for more details. 38 | """ 39 | _script = tmp_path / "gen_certs.sh" 40 | _ca_dir = tmp_path / "root_ca" 41 | _interm_dir = tmp_path / "interm_ca" 42 | _certs_dir = tmp_path / "certs" 43 | 44 | _ca_dir.mkdir() 45 | _interm_dir.mkdir() 46 | _certs_dir.mkdir() 47 | 48 | shutil.copy(CERT_GEN_SCRIPT, _script) 49 | os.chmod(_script, 0o750) 50 | for chain in CHAINS: 51 | subprocess.run([str(_script), chain], check=True, capture_output=True) 52 | for _k in tmp_path.glob("*.key"): 53 | _k.unlink() 54 | shutil.move(f"{chain}.root.pem", _ca_dir) 55 | shutil.move(f"{chain}.interm.pem", _interm_dir) 56 | shutil.move("sign.pem", _certs_dir / f"{chain}.sign.pem") 57 | 58 | logger.info(f"CA {CHAINS} generated") 59 | return _ca_dir, _interm_dir, _certs_dir 60 | 61 | 62 | @pytest.mark.parametrize( 63 | "chain", 64 | ( 65 | ("dev"), 66 | ("stg"), 67 | ("prd"), 68 | ), 69 | ) 70 | def test_ca_store_map(chain, gen_ca_chains: tuple[Path, Path, Path]): 71 | _ca_dir, _interm_dir, _certs_dir = gen_ca_chains 72 | ca_stores = load_ca_store(_ca_dir) 73 | 74 | _cert = load_pem_x509_certificate((_certs_dir / f"{chain}.sign.pem").read_bytes()) 75 | _interm = load_pem_x509_certificate( 76 | (_interm_dir / f"{chain}.interm.pem").read_bytes() 77 | ) 78 | ca_stores.verify(_cert, interm_cas=[_interm]) 79 | -------------------------------------------------------------------------------- /src/ota_proxy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | 20 | from otaclient._utils import SharedOTAClientMetricsWriter 21 | 22 | from .cache_control_header import OTAFileCacheControl 23 | from .config import config 24 | from .ota_cache import OTACache 25 | from .server_app import App 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | __all__ = ( 31 | "App", 32 | "OTACache", 33 | "OTAFileCacheControl", 34 | "config", 35 | ) 36 | 37 | 38 | def run_otaproxy( 39 | host: str, 40 | port: int, 41 | *, 42 | init_cache: bool, 43 | cache_dir: str = config.BASE_DIR, 44 | cache_db_f: str = config.DB_FILE, 45 | upper_proxy: str, 46 | enable_cache: bool, 47 | enable_https: bool, 48 | external_cache_mnt_point: str | None = None, 49 | external_nfs_cache_mnt_point: str | None = None, 50 | shm_metrics_writer: SharedOTAClientMetricsWriter | None = None, 51 | ): 52 | import asyncio 53 | 54 | import anyio 55 | import uvicorn 56 | import uvloop 57 | 58 | from . import App, OTACache 59 | 60 | _ota_cache = OTACache( 61 | base_dir=cache_dir, 62 | db_file=cache_db_f, 63 | cache_enabled=enable_cache, 64 | upper_proxy=upper_proxy, 65 | enable_https=enable_https, 66 | init_cache=init_cache, 67 | external_cache_mnt_point=external_cache_mnt_point, 68 | external_nfs_cache_mnt_point=external_nfs_cache_mnt_point, 69 | shm_metrics_writer=shm_metrics_writer, 70 | ) 71 | _config = uvicorn.Config( 72 | App(_ota_cache), 73 | host=host, 74 | port=port, 75 | log_level="error", 76 | lifespan="on", 77 | loop="uvloop", 78 | # NOTE: must use h11, other http implementation will break HTTP proxy 79 | http="h11", 80 | ) 81 | _server = uvicorn.Server(_config) 82 | 83 | asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 84 | anyio.run( 85 | _server.serve, 86 | backend="asyncio", 87 | backend_options={"loop_factory": uvloop.new_event_loop}, 88 | ) 89 | -------------------------------------------------------------------------------- /src/otaclient/configs/cfg.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Load ecu_info, proxy_info and otaclient configs.""" 15 | 16 | from typing import TYPE_CHECKING, Any 17 | 18 | from otaclient.configs._cfg_configurable import ConfigurableSettings, set_configs 19 | from otaclient.configs._cfg_consts import Consts 20 | from otaclient.configs._ecu_info import ( 21 | BootloaderType, 22 | ECUContact, 23 | ECUInfo, 24 | parse_ecu_info, 25 | ) 26 | from otaclient.configs._proxy_info import ProxyInfo, parse_proxy_info 27 | 28 | __all__ = [ 29 | "BootloaderType", 30 | "ECUContact", 31 | "ECUInfo", 32 | "ecu_info", 33 | "ProxyInfo", 34 | "proxy_info", 35 | "cfg", 36 | ] 37 | 38 | cfg_configurable = set_configs() 39 | cfg_consts = Consts() 40 | 41 | ECU_INFO_LOADED_SUCCESSFULLY: bool 42 | """A const set at startup time ecu_info.yaml parsing. 43 | 44 | If it is False, it means that the ecu_info.yaml file is invalid, 45 | and the default ecu_info(defined in _ecu_info module) is used. 46 | """ 47 | PROXY_INFO_LOADED_SUCCESSFULLY: bool 48 | """A const set at startup time proxy_info.yaml parsing. 49 | 50 | If it is False, it means that the proxy_info.yaml file is invalid, 51 | and the default proxy_info(defined in _proxy_info module) is used. 52 | """ 53 | 54 | if TYPE_CHECKING: 55 | 56 | class _OTAClientConfigs(ConfigurableSettings, Consts): 57 | """OTAClient configs.""" 58 | 59 | else: 60 | 61 | class _OTAClientConfigs: 62 | 63 | # NOTE(20241108): still use __getattr__ to allow changing/mocking attributes 64 | # for easy testing. 65 | def __getattr__(self, name: str) -> Any: 66 | for _cfg in [cfg_consts, cfg_configurable]: 67 | try: 68 | return getattr(_cfg, name) 69 | except AttributeError: 70 | continue 71 | raise AttributeError(f"no such config field: {name=}") 72 | 73 | 74 | cfg = _OTAClientConfigs() 75 | ECU_INFO_LOADED_SUCCESSFULLY, ecu_info = parse_ecu_info( 76 | ecu_info_file=cfg.ECU_INFO_FPATH 77 | ) 78 | PROXY_INFO_LOADED_SUCCESSFULLY, proxy_info = parse_proxy_info( 79 | proxy_info_file=cfg.PROXY_INFO_FPATH 80 | ) 81 | -------------------------------------------------------------------------------- /src/otaclient/ota_core/_download_bsp_version_file.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Download BSP version file.""" 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | import time 20 | from http import HTTPStatus 21 | from typing import Optional 22 | from urllib.parse import urlsplit 23 | 24 | from otaclient.boot_control.configs import JetsonBootCommon 25 | from otaclient_common.downloader import DownloaderPool 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | BSP_VERSION_PATH = "data" + JetsonBootCommon.NV_TEGRA_RELEASE_FPATH 30 | DOWNLOAD_TIMEOUT = 2 31 | RETRY_TIMES = 12 32 | RETRY_INTERVAL = 1 # second 33 | 34 | 35 | # API function 36 | def download(base_url: str, *, downloader_pool: DownloaderPool) -> Optional[str]: 37 | """Download BSP version file content.""" 38 | logger.info("perform BSP version compatibility check ...") 39 | _downloader = downloader_pool.get_instance() 40 | try: 41 | _hint_file_url = f"{base_url.rstrip('/')}/{BSP_VERSION_PATH}" 42 | if _downloader._force_http: 43 | _hint_file_url = urlsplit(_hint_file_url)._replace(scheme="http").geturl() 44 | 45 | for _ in range(RETRY_TIMES): 46 | try: 47 | resp = _downloader._session.get( 48 | _hint_file_url, timeout=DOWNLOAD_TIMEOUT 49 | ) 50 | _status_code = resp.status_code 51 | if _status_code == HTTPStatus.OK: 52 | logger.info("BSP version file downloaded successfully.") 53 | return resp.text.strip() 54 | if _status_code in [HTTPStatus.UNAUTHORIZED, HTTPStatus.NOT_FOUND]: 55 | logger.info("BSP version file is unauthorized or not found.") 56 | return None 57 | except Exception: 58 | pass 59 | 60 | time.sleep(RETRY_INTERVAL) 61 | logger.info("BSP version file could not be downloaded within retry limit.") 62 | return None 63 | except Exception as e: 64 | logger.warning(f"unexpected failure during probing image version: {e}") 65 | return None 66 | finally: 67 | downloader_pool.release_instance() 68 | -------------------------------------------------------------------------------- /src/otaclient_common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Common shared libs for otaclient.""" 15 | 16 | from __future__ import annotations 17 | 18 | import os 19 | from math import ceil 20 | from pathlib import Path 21 | from typing import Optional 22 | 23 | from typing_extensions import Literal 24 | 25 | _MultiUnits = Literal["GiB", "MiB", "KiB", "Bytes", "KB", "MB", "GB"] 26 | # fmt: off 27 | _multiplier: dict[_MultiUnits, int] = { 28 | "GiB": 1024 ** 3, "MiB": 1024 ** 2, "KiB": 1024 ** 1, 29 | "GB": 1000 ** 3, "MB": 1000 ** 2, "KB": 1000 ** 1, 30 | "Bytes": 1, 31 | } 32 | # fmt: on 33 | 34 | 35 | def get_file_size( 36 | swapfile_fpath: str | Path, units: _MultiUnits = "Bytes" 37 | ) -> Optional[int]: 38 | """Helper for get file size with .""" 39 | swapfile_fpath = Path(swapfile_fpath) 40 | if swapfile_fpath.is_file(): 41 | return ceil(swapfile_fpath.stat().st_size / _multiplier[units]) 42 | 43 | 44 | def human_readable_size(_in: int) -> str: 45 | for _mu_name, _mu in _multiplier.items(): 46 | if _mu == 1: 47 | break 48 | if (_res := (_in / _mu)) > 1: 49 | return f"{_res:.2f} {_mu_name}" 50 | return f"{_in} Bytes" 51 | 52 | 53 | def replace_root( 54 | path: str | Path, old_root: str | Path | Literal["/"], new_root: str | Path 55 | ) -> str: 56 | """Replace a relative to to . 57 | 58 | For example, if path="/abc", old_root="/", new_root="/new_root", 59 | then we will have "/new_root/abc". 60 | """ 61 | old_root, new_root = str(old_root), str(new_root) 62 | if not (old_root.startswith("/") and new_root.startswith("/")): 63 | raise ValueError(f"{old_root=} and/or {new_root=} is not valid root") 64 | if os.path.commonpath([path, old_root]) != old_root: 65 | raise ValueError(f"{old_root=} is not the root of {path=}") 66 | return os.path.join(new_root, os.path.relpath(path, old_root)) 67 | 68 | 69 | EMPTY_FILE_SHA256 = r"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 70 | EMPTY_FILE_SHA256_BYTE = bytes.fromhex( 71 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 72 | ) 73 | SHA256DIGEST_HEX_LEN = len(EMPTY_FILE_SHA256) 74 | SHA256DIGEST_LEN = len(EMPTY_FILE_SHA256_BYTE) 75 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_proto_wrapper/example_pb2_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from typing import Iterable as _Iterable 19 | from typing import Mapping as _Mapping 20 | from typing import Optional as _Optional 21 | from typing import Union as _Union 22 | 23 | from otaclient_common.proto_wrapper import ( 24 | Duration, 25 | EnumWrapper, 26 | MessageMapContainer, 27 | MessageWrapper, 28 | RepeatedCompositeContainer, 29 | RepeatedScalarContainer, 30 | ScalarMapContainer, 31 | calculate_slots, 32 | ) 33 | 34 | from . import example_pb2 as _pb2 35 | 36 | 37 | class SampleEnum(EnumWrapper): 38 | VALUE_0 = _pb2.VALUE_0 39 | VALUE_1 = _pb2.VALUE_1 40 | VALUE_2 = _pb2.VALUE_2 41 | 42 | 43 | class InnerMessage(MessageWrapper[_pb2.InnerMessage]): 44 | __slots__ = calculate_slots(_pb2.InnerMessage) 45 | duration_field: Duration 46 | enum_field: SampleEnum 47 | double_field: float 48 | int_field: int 49 | str_field: str 50 | 51 | def __init__( 52 | self, 53 | int_field: _Optional[int] = ..., 54 | double_field: _Optional[float] = ..., 55 | str_field: _Optional[str] = ..., 56 | duration_field: _Optional[_Union[Duration, _Mapping]] = ..., 57 | enum_field: _Optional[_Union[SampleEnum, str]] = ..., 58 | ) -> None: ... 59 | 60 | 61 | class OuterMessage(MessageWrapper[_pb2.OuterMessage]): 62 | __slots__ = calculate_slots(_pb2.OuterMessage) 63 | mapping_composite_field: MessageMapContainer[int, InnerMessage] 64 | mapping_scalar_field: ScalarMapContainer[str, str] 65 | nested_msg: InnerMessage 66 | repeated_composite_field: RepeatedCompositeContainer[InnerMessage] 67 | repeated_scalar_field: RepeatedScalarContainer[str] 68 | 69 | def __init__( 70 | self, 71 | repeated_scalar_field: _Optional[_Iterable[str]] = ..., 72 | repeated_composite_field: _Optional[ 73 | _Iterable[_Union[InnerMessage, _Mapping]] 74 | ] = ..., 75 | nested_msg: _Optional[_Union[InnerMessage, _Mapping]] = ..., 76 | mapping_scalar_field: _Optional[_Mapping[str, str]] = ..., 77 | mapping_composite_field: _Optional[_Mapping[int, InnerMessage]] = ..., 78 | ) -> None: ... 79 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_proto_wrapper/example_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: example.proto 4 | """Generated protocol buffer code.""" 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import descriptor_pool as _descriptor_pool 7 | from google.protobuf import symbol_database as _symbol_database 8 | from google.protobuf.internal import builder as _builder 9 | 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 16 | 17 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( 18 | b'\n\rexample.proto\x1a\x1egoogle/protobuf/duration.proto"\x9e\x01\n\x0cInnerMessage\x12\x11\n\tint_field\x18\x01 \x01(\r\x12\x14\n\x0c\x64ouble_field\x18\x02 \x01(\x01\x12\x11\n\tstr_field\x18\x03 \x01(\t\x12\x31\n\x0e\x64uration_field\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1f\n\nenum_field\x18\x05 \x01(\x0e\x32\x0b.SampleEnum"\x99\x03\n\x0cOuterMessage\x12\x1d\n\x15repeated_scalar_field\x18\x01 \x03(\t\x12/\n\x18repeated_composite_field\x18\x02 \x03(\x0b\x32\r.InnerMessage\x12!\n\nnested_msg\x18\x03 \x01(\x0b\x32\r.InnerMessage\x12\x43\n\x14mapping_scalar_field\x18\x04 \x03(\x0b\x32%.OuterMessage.MappingScalarFieldEntry\x12I\n\x17mapping_composite_field\x18\x05 \x03(\x0b\x32(.OuterMessage.MappingCompositeFieldEntry\x1a\x39\n\x17MappingScalarFieldEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aK\n\x1aMappingCompositeFieldEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12\x1c\n\x05value\x18\x02 \x01(\x0b\x32\r.InnerMessage:\x02\x38\x01*3\n\nSampleEnum\x12\x0b\n\x07VALUE_0\x10\x00\x12\x0b\n\x07VALUE_1\x10\x01\x12\x0b\n\x07VALUE_2\x10\x02\x62\x06proto3' 19 | ) 20 | 21 | _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) 22 | _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "example_pb2", globals()) 23 | if _descriptor._USE_C_DESCRIPTORS == False: 24 | 25 | DESCRIPTOR._options = None 26 | _OUTERMESSAGE_MAPPINGSCALARFIELDENTRY._options = None 27 | _OUTERMESSAGE_MAPPINGSCALARFIELDENTRY._serialized_options = b"8\001" 28 | _OUTERMESSAGE_MAPPINGCOMPOSITEFIELDENTRY._options = None 29 | _OUTERMESSAGE_MAPPINGCOMPOSITEFIELDENTRY._serialized_options = b"8\001" 30 | _SAMPLEENUM._serialized_start = 622 31 | _SAMPLEENUM._serialized_end = 673 32 | _INNERMESSAGE._serialized_start = 50 33 | _INNERMESSAGE._serialized_end = 208 34 | _OUTERMESSAGE._serialized_start = 211 35 | _OUTERMESSAGE._serialized_end = 620 36 | _OUTERMESSAGE_MAPPINGSCALARFIELDENTRY._serialized_start = 486 37 | _OUTERMESSAGE_MAPPINGSCALARFIELDENTRY._serialized_end = 543 38 | _OUTERMESSAGE_MAPPINGCOMPOSITEFIELDENTRY._serialized_start = 545 39 | _OUTERMESSAGE_MAPPINGCOMPOSITEFIELDENTRY._serialized_end = 620 40 | # @@protoc_insertion_point(module_scope) 41 | -------------------------------------------------------------------------------- /tests/test_otaclient/test_create_standby/test_resume_ota.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | import logging 18 | import queue 19 | from pathlib import Path 20 | from typing import cast 21 | 22 | import pytest 23 | import pytest_mock 24 | from ota_image_libs.v1.file_table.db import FileTableDBHelper 25 | 26 | from otaclient.create_standby._common import ResourcesDigestWithSize 27 | from otaclient.create_standby.delta_gen import InPlaceDeltaWithBaseFileTable 28 | from otaclient.create_standby.resume_ota import ResourceScanner 29 | 30 | from .conftest import SlotAB 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | DELTA_GEN_MODULE = "otaclient.create_standby.delta_gen" 35 | DELTA_GEN_FULL_DISK_SCAN_BASE = f"{DELTA_GEN_MODULE}.DeltaGenFullDiskScan" 36 | 37 | 38 | class _MockedQue: 39 | @staticmethod 40 | def put_nowait(_): ... 41 | 42 | 43 | MockedQue = cast(queue.Queue, _MockedQue) 44 | 45 | 46 | @pytest.fixture(autouse=True) 47 | def mocked_full_disk_scan_mode(mocker: pytest_mock.MockerFixture): 48 | # NOTE: to let full disk scan mode can get all the resources, only for test 49 | mocker.patch(f"{DELTA_GEN_FULL_DISK_SCAN_BASE}.EXCLUDE_PATHS", {}) 50 | mocker.patch(f"{DELTA_GEN_MODULE}.MAX_FOLDER_DEEPTH", 2**32) 51 | mocker.patch(f"{DELTA_GEN_MODULE}.MAX_FILENUM_PER_FOLDER", 2**64) 52 | 53 | 54 | def test_resume_ota( 55 | ab_slots_for_inplace: SlotAB, 56 | fst_db_helper: FileTableDBHelper, 57 | resource_dir: Path, 58 | ) -> None: 59 | logger.info("process slot and generating OTA resources ...") 60 | _all_digests = ResourcesDigestWithSize.from_iterable( 61 | fst_db_helper.select_all_digests_with_size() 62 | ) 63 | 64 | InPlaceDeltaWithBaseFileTable( 65 | file_table_db_helper=fst_db_helper, 66 | all_resource_digests=_all_digests, 67 | delta_src=ab_slots_for_inplace.slot_b, 68 | copy_dst=resource_dir, 69 | status_report_queue=MockedQue, 70 | session_id="session_id", 71 | ).process_slot(str(fst_db_helper.db_f)) 72 | 73 | logger.info("scan through the generated resources ...") 74 | ResourceScanner( 75 | all_resource_digests=_all_digests, 76 | resource_dir=resource_dir, 77 | status_report_queue=MockedQue, 78 | session_id="session_id", 79 | ).resume_ota() 80 | -------------------------------------------------------------------------------- /docker/mini_ota_img/new_ota_image.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG SYS_IMG=ghcr.io/tier4/ota-client/sys_img_for_test:ubuntu_22.04 2 | ARG UBUNTU_BASE=ubuntu:22.04 3 | 4 | # 5 | # ------ stage 1: prepare base image ------ # 6 | # 7 | 8 | FROM ${SYS_IMG} AS sys_img 9 | 10 | 11 | # 12 | # ------ stage 2: build new OTA image ------ # 13 | # 14 | 15 | FROM ${UBUNTU_BASE} AS ota_img_builder 16 | 17 | ARG OTA_IMAGE_BUILDER_RELEASE="https://github.com/tier4/ota-image-builder/releases/download/v0.3.2/ota-image-builder-x86_64" 18 | 19 | SHELL ["/bin/bash", "-c"] 20 | ENV DEBIAN_FRONTEND=noninteractive 21 | 22 | ENV OTA_IMAGE_BUILDER_RELEASE=${OTA_IMAGE_BUILDER_RELEASE} 23 | ENV OTA_IMAGE_SERVER_ROOT="/ota-image" 24 | ENV ROOTFS="/rootfs" 25 | ENV CERTS_DIR="/certs" 26 | ENV BUILD_ROOT="/tmp/build_root" 27 | 28 | COPY --chmod=755 ./tests/keys/gen_certs.sh ${CERTS_DIR}/gen_certs.sh 29 | COPY ./tests/data/ota_image_builder ${BUILD_ROOT} 30 | 31 | WORKDIR ${BUILD_ROOT} 32 | 33 | RUN --mount=type=bind,source=/,target=/rootfs,from=sys_img,rw \ 34 | set -eux; \ 35 | apt-get update -qq; \ 36 | apt-get install -y -qq --no-install-recommends \ 37 | ca-certificates wget; \ 38 | apt-get clean; \ 39 | rm -rf /var/lib/apt/lists/*; \ 40 | wget ${OTA_IMAGE_BUILDER_RELEASE} -O ${BUILD_ROOT}/ota-image-builder; \ 41 | chmod +x ota-image-builder; \ 42 | # --- generate certs --- # 43 | pushd ${CERTS_DIR}; \ 44 | bash ${CERTS_DIR}/gen_certs.sh; \ 45 | popd; \ 46 | # --- start to build the OTA image --- # 47 | mkdir -p ${OTA_IMAGE_SERVER_ROOT}; \ 48 | ./ota-image-builder -d init \ 49 | --annotations-file full_annotations.yaml \ 50 | ${OTA_IMAGE_SERVER_ROOT}; \ 51 | ./ota-image-builder -d add-image \ 52 | --annotations-file full_annotations.yaml \ 53 | --release-key dev \ 54 | --sys-config "autoware:sys_config.yaml" \ 55 | --rootfs ${ROOTFS} \ 56 | ${OTA_IMAGE_SERVER_ROOT}; \ 57 | ./ota-image-builder -d add-image \ 58 | --annotations-file full_annotations.yaml \ 59 | --release-key prd \ 60 | --sys-config "autoware:sys_config.yaml" \ 61 | --rootfs ${ROOTFS}/var \ 62 | ${OTA_IMAGE_SERVER_ROOT}; \ 63 | ./ota-image-builder -d finalize ${OTA_IMAGE_SERVER_ROOT}; \ 64 | ./ota-image-builder -d sign \ 65 | --sign-cert ${CERTS_DIR}/sign.pem \ 66 | --sign-key ${CERTS_DIR}/sign.key \ 67 | --ca-cert ${CERTS_DIR}/test.interm.pem \ 68 | ${OTA_IMAGE_SERVER_ROOT}; \ 69 | # --- clean up --- # 70 | # although the keys are only for tests, and only used for 71 | # this test OTA image, still clean it up. 72 | rm -rf ${CERTS_DIR}/*.key 73 | 74 | # 75 | # ------ stage 4: output OTA image ------ # 76 | # 77 | 78 | # NOTE: use busybox so that the overall image size will not increase much, 79 | # while enable us to examine the built OTA image. 80 | 81 | FROM ${UBUNTU_BASE} 82 | 83 | ARG SYS_IMG 84 | 85 | LABEL org.opencontainers.image.description=\ 86 | "A mini OTA image v1 OTA image for OTAClient test, based on system image ${SYS_IMG}" 87 | 88 | COPY --from=ota_img_builder /ota-image /ota-image 89 | COPY --from=ota_img_builder /certs /certs 90 | -------------------------------------------------------------------------------- /tests/test_ota_proxy/test_cache_control_header.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from typing import Any, Dict 17 | 18 | import pytest 19 | 20 | from ota_proxy import OTAFileCacheControl 21 | 22 | 23 | @pytest.mark.parametrize( 24 | "raw_str, expected", 25 | ( 26 | ("no_cache", OTAFileCacheControl(no_cache=True)), 27 | ("retry_caching", OTAFileCacheControl(retry_caching=True)), 28 | ( 29 | "no_cache, file_sha256=sha256value, file_compression_alg=zst", 30 | OTAFileCacheControl( 31 | no_cache=True, 32 | file_sha256="sha256value", 33 | file_compression_alg="zst", 34 | ), 35 | ), 36 | ), 37 | ) 38 | def test__parse_header(raw_str, expected): 39 | assert OTAFileCacheControl.parse_header(raw_str) == expected 40 | 41 | 42 | @pytest.mark.parametrize( 43 | "kwargs, expected", 44 | ( 45 | ({"no_cache": True, "retry_caching": False}, "no_cache"), 46 | ( 47 | {"no_cache": True, "file_sha256": "sha256_value"}, 48 | "no_cache,file_sha256=sha256_value", 49 | ), 50 | ( 51 | { 52 | "retry_caching": True, 53 | "file_sha256": "sha256_value", 54 | "file_compression_alg": "zst", 55 | }, 56 | "retry_caching,file_sha256=sha256_value,file_compression_alg=zst", 57 | ), 58 | ), 59 | ) 60 | def test__export_kwargs_as_header(kwargs: Dict[str, Any], expected: str): 61 | assert OTAFileCacheControl.export_kwargs_as_header(**kwargs) == expected 62 | 63 | 64 | @pytest.mark.parametrize( 65 | "_input, kwargs, expected", 66 | ( 67 | ( 68 | "file_sha256=sha256_value,file_compression_alg=zst", 69 | {"no_cache": True, "retry_caching": True}, 70 | "file_sha256=sha256_value,file_compression_alg=zst,no_cache,retry_caching", 71 | ), 72 | ( 73 | "file_sha256=sha256_value,file_compression_alg=zst", 74 | {"file_sha256": "new_sha256_value", "retry_caching": True}, 75 | "file_sha256=new_sha256_value,file_compression_alg=zst,retry_caching", 76 | ), 77 | ( 78 | "retry_caching,file_sha256=sha256_value,file_compression_alg=zst", 79 | {"retry_caching": False}, 80 | "file_sha256=sha256_value,file_compression_alg=zst", 81 | ), 82 | ), 83 | ) 84 | def test__update_header_str(_input: str, kwargs: Dict[str, Any], expected: str): 85 | assert OTAFileCacheControl.update_header_str(_input, **kwargs) == expected 86 | -------------------------------------------------------------------------------- /src/ota_metadata/legacy2/rs_table.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Resource table implementation for legacy OTA image.""" 15 | 16 | 17 | from __future__ import annotations 18 | 19 | import random 20 | from typing import Any, ClassVar, Generator, Literal, Optional 21 | 22 | from pydantic import SkipValidation 23 | from simple_sqlite3_orm import ( 24 | ConstrainRepr, 25 | CreateTableParams, 26 | ORMBase, 27 | ORMThreadPoolBase, 28 | TableSpec, 29 | ) 30 | from typing_extensions import Annotated, Self 31 | 32 | RSTABLE_NAME = "rs_table" 33 | 34 | 35 | class ResourceTable(TableSpec): 36 | schema_ver: ClassVar[Literal[1]] = 1 37 | 38 | digest: Annotated[ 39 | bytes, 40 | ConstrainRepr("PRIMARY KEY"), 41 | SkipValidation, 42 | ] 43 | """sha256 digest of the original file.""" 44 | 45 | path: Annotated[ 46 | Optional[str], 47 | SkipValidation, 48 | ] = None 49 | """NOTE: only for resource without zstd compression.""" 50 | 51 | original_size: Annotated[ 52 | int, 53 | ConstrainRepr("NOT NULL"), 54 | SkipValidation, 55 | ] 56 | """The size of the plain uncompressed resource.""" 57 | 58 | compression_alg: Annotated[ 59 | Optional[str], 60 | SkipValidation, 61 | ] = None 62 | """The compression algorthim used to compressed the resource. 63 | 64 | NOTE that this field should be None if is not None. 65 | """ 66 | 67 | def __eq__(self, other: Any | Self) -> bool: 68 | return isinstance(other, self.__class__) and self.digest == other.digest 69 | 70 | def __hash__(self) -> int: 71 | return hash(self.digest) 72 | 73 | 74 | class ResourceTableORM(ORMBase[ResourceTable]): 75 | 76 | orm_bootstrap_table_name = RSTABLE_NAME 77 | orm_bootstrap_create_table_params = CreateTableParams(without_rowid=True) 78 | 79 | def iter_all_with_shuffle(self, *, batch_size: int) -> Generator[ResourceTable]: 80 | """Iter all entries with seek method by rowid, shuffle each batch before yield. 81 | 82 | NOTE: the target table must has rowid defined! 83 | """ 84 | _this_batch = [] 85 | for _entry in self.orm_select_entries(): 86 | _this_batch.append(_entry) 87 | if len(_this_batch) >= batch_size: 88 | random.shuffle(_this_batch) 89 | yield from _this_batch 90 | _this_batch = [] 91 | random.shuffle(_this_batch) 92 | yield from _this_batch 93 | 94 | 95 | class ResourceTableORMPool(ORMThreadPoolBase[ResourceTable]): 96 | 97 | orm_bootstrap_table_name = RSTABLE_NAME 98 | -------------------------------------------------------------------------------- /.github/actions/build_patches/action.yml: -------------------------------------------------------------------------------- 1 | name: "Build SquashFS Image" 2 | description: "Builds a SquashFS image using Docker" 3 | inputs: 4 | platform_suffix: 5 | description: "The platform suffix" 6 | required: true 7 | version: 8 | description: "The OTA client version" 9 | required: true 10 | squashfs: 11 | description: "The target SquashFS file" 12 | required: true 13 | output_dir: 14 | description: "The output directory" 15 | required: true 16 | runs: 17 | using: "composite" 18 | steps: 19 | - name: set self version as environment variable 20 | env: 21 | VERSION: ${{ inputs.version }} 22 | shell: bash 23 | run: | 24 | MAJOR=$(echo ${VERSION} | cut -d. -f1) 25 | MINOR=$(echo ${VERSION} | cut -d. -f2) 26 | PATCH=$(echo ${VERSION} | cut -d. -f3) 27 | 28 | echo "MAJOR=${MAJOR:-0}" >> $GITHUB_ENV 29 | echo "MINOR=${MINOR:-0}" >> $GITHUB_ENV 30 | echo "PATCH=${PATCH:-0}" >> $GITHUB_ENV 31 | 32 | - name: get released versions 33 | shell: bash 34 | env: 35 | GH_TOKEN: ${{ github.token }} 36 | run: | 37 | RELEASED_VERSIONS=$(gh release list --repo tier4/ota-client \ 38 | --exclude-pre-releases --json tagName \ 39 | --jq '.[].tagName' | \ 40 | sed 's/^v//' | \ 41 | tr '\n' ' ') 42 | echo "RELEASED_VERSIONS=${RELEASED_VERSIONS}" >> $GITHUB_ENV 43 | 44 | - name: extract target versions 45 | env: 46 | RELEASED_VERSIONS: ${{ env.RELEASED_VERSIONS }} 47 | MAJOR: ${{ env.MAJOR }} 48 | MINOR: ${{ env.MINOR }} 49 | PATCH: ${{ env.PATCH }} 50 | shell: bash 51 | # extract versions that meet the the following conditions: 52 | # 1. include the same major version and the same or the previous minor version 53 | # 2. exclude the same major, minor, and patch version 54 | run: | 55 | BASE_VERSIONS=$(echo "${RELEASED_VERSIONS}" | \ 56 | tr ' ' '\n' | \ 57 | grep -E "^${MAJOR}\.(${MINOR}|$(( ${MINOR} - 1)))\.[0-9]+$" | \ 58 | grep -vE "^${MAJOR}\.${MINOR}\.${PATCH}$" | \ 59 | tr '\n' ' ' || true) 60 | echo "BASE_VERSIONS=${BASE_VERSIONS}" >> $GITHUB_ENV 61 | 62 | - name: download and create patches for target versions 63 | env: 64 | BASE_VERSIONS: ${{ env.BASE_VERSIONS }} 65 | PLATFORM_SUFFIX: ${{ inputs.platform_suffix }} 66 | VERSION: ${{ inputs.version }} 67 | SQUASHFS: ${{ inputs.squashfs }} 68 | OUTPUT_DIR: ${{ inputs.output_dir }} 69 | shell: bash 70 | # if the release version contains "otaclient-${platform_suffix}_v${version}.squashfs" asset, 71 | # download the squashfs file, create a patch file then save it to the output directory 72 | run: | 73 | for BASE_VERSION in ${BASE_VERSIONS}; do 74 | BASE_SQUASHFS="otaclient-${PLATFORM_SUFFIX}_v${BASE_VERSION}.squashfs" 75 | ASSET_URL="https://github.com/tier4/ota-client/releases/download/v${BASE_VERSION}/${BASE_SQUASHFS}" 76 | if curl --output /dev/null --silent --head --fail "${ASSET_URL}"; then 77 | 78 | curl -L -o ${BASE_SQUASHFS} ${ASSET_URL} || continue 79 | 80 | PATCH_FILE="otaclient-${PLATFORM_SUFFIX}_v${BASE_VERSION}-v${VERSION}.patch" 81 | zstd --patch-from=${BASE_SQUASHFS} ${SQUASHFS} -o ${OUTPUT_DIR}/${PATCH_FILE} || continue 82 | fi 83 | done 84 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_ota_image_v1/test_e2e.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | import logging 18 | import os 19 | from hashlib import sha256 20 | from pathlib import Path 21 | from queue import Queue 22 | 23 | from ota_image_libs.v1.image_manifest.schema import ImageIdentifier, OTAReleaseKey 24 | from pytest_mock import MockerFixture 25 | 26 | from ota_metadata.utils.cert_store import load_ca_store 27 | from otaclient._types import MultipleECUStatusFlags 28 | from otaclient._utils import SharedOTAClientMetricsReader 29 | from otaclient.metrics import OTAMetricsData 30 | from otaclient.ota_core._updater_base import OTAImageV1SupportMixin 31 | from otaclient_common.downloader import DownloaderPool 32 | from tests.conftest import cfg 33 | from tests.test_ota_metadata.conftest import iter_helper 34 | 35 | logger = logging.getLogger(__name__) 36 | 37 | 38 | def test_download_and_parse_metadata(tmp_path: Path, mocker: MockerFixture): 39 | # ------ execution ------ # 40 | # NOTE: directly bootstrap the mixin as we only check metadata downloading and parsing 41 | ota_image_v1 = OTAImageV1SupportMixin( 42 | version="dummy_version", 43 | raw_url_base=cfg.OTA_IMAGE_V1_URL, 44 | session_wd=tmp_path, 45 | downloader_pool=DownloaderPool(instance_num=3, hash_func=sha256), 46 | session_id=f"session_id_{os.urandom(2).hex()}", 47 | ecu_status_flags=mocker.MagicMock(spec=MultipleECUStatusFlags), 48 | status_report_queue=mocker.MagicMock(spec=Queue), 49 | metrics=mocker.MagicMock(spec=OTAMetricsData), 50 | shm_metrics_reader=mocker.MagicMock(spec=SharedOTAClientMetricsReader), 51 | ) # type: ignore 52 | 53 | ca_store = load_ca_store(cfg.CERTS_OTA_IMAGE_V1_DIR) 54 | ota_image_v1.setup_ota_image_support( 55 | ca_store=ca_store, 56 | image_identifier=ImageIdentifier("autoware", OTAReleaseKey.dev), 57 | ) 58 | ota_image_v1._process_metadata() 59 | 60 | # ------ check result ------ # 61 | ota_image_helper = ota_image_v1._ota_image_helper 62 | assert (_image_index := ota_image_helper.image_index) 63 | assert (_image_manifest := ota_image_helper.image_manifest) 64 | assert (_image_config := ota_image_helper.image_config) 65 | logger.info(str(_image_index)) 66 | logger.info(str(_image_manifest)) 67 | logger.info(str(_image_config)) 68 | 69 | fst_helper = ota_image_helper.file_table_helper 70 | assert ( 71 | iter_helper(fst_helper.iter_dir_entries()) 72 | == _image_config.labels.sys_image_dirs_count 73 | ) 74 | assert ( 75 | iter_helper(fst_helper.iter_non_regular_entries()) 76 | == _image_config.labels.sys_image_non_regular_files_count 77 | ) 78 | assert ( 79 | iter_helper(fst_helper.iter_regular_entries()) 80 | == _image_config.sys_image_regular_files_count 81 | ) 82 | -------------------------------------------------------------------------------- /docker/mini_ota_img/ota_image.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG SYS_IMG=ghcr.io/tier4/ota-client/sys_img_for_test:ubuntu_22.04 2 | ARG UBUNTU_BASE=ubuntu:22.04 3 | ARG BUSYBOX_VER=busybox:1.37.0 4 | 5 | # 6 | # ------ stage 1: prepare base image ------ # 7 | # 8 | 9 | FROM ${SYS_IMG} AS sys_img 10 | 11 | # 12 | # ------ stage 2: prepare OTA image build environment ------ # 13 | # 14 | FROM ${UBUNTU_BASE} AS ota_img_builder 15 | 16 | SHELL ["/bin/bash", "-c"] 17 | ENV DEBIAN_FRONTEND=noninteractive 18 | 19 | ENV OTA_METADATA_REPO="https://github.com/tier4/ota-metadata" 20 | ENV OTA_IMAGE_SERVER_ROOT="/ota-image" 21 | ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data" 22 | ENV CERTS_DIR="/certs" 23 | ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" 24 | 25 | COPY --from=sys_img / /${OTA_IMAGE_DIR} 26 | COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/ 27 | 28 | WORKDIR ${OTA_IMAGE_SERVER_ROOT} 29 | 30 | RUN set -eux; \ 31 | apt-get update -qq; \ 32 | apt-get install -y -qq --no-install-recommends \ 33 | gcc \ 34 | git \ 35 | libcurl4-openssl-dev \ 36 | libssl-dev \ 37 | python3-dev \ 38 | python3-minimal \ 39 | python3-pip \ 40 | python3-venv \ 41 | wget; \ 42 | apt-get install -y -qq linux-image-generic; \ 43 | apt-get clean; \ 44 | # install uv 45 | python3 -m pip install --no-cache-dir -q -U pip; \ 46 | python3 -m pip install --no-cache-dir uv; \ 47 | # generate keys and certs for signing 48 | mkdir -p "${CERTS_DIR}"; \ 49 | pushd /tmp/certs; \ 50 | ./gen_certs.sh; \ 51 | mv ./* "${CERTS_DIR}"; \ 52 | popd; \ 53 | cp "${CERTS_DIR}"/sign.pem sign.pem; \ 54 | # git clone the ota-metadata repository 55 | git clone ${OTA_METADATA_REPO}; \ 56 | pushd ota-metadata; git checkout 6c8ad47; popd; \ 57 | # prepare build environment 58 | python3 -m venv ota-metadata/.venv; \ 59 | source ota-metadata/.venv/bin/activate; \ 60 | python3 -m pip install --no-cache-dir -U pip; \ 61 | python3 -m pip install --no-cache-dir -q \ 62 | -r ota-metadata/metadata/ota_metadata/requirements.txt; \ 63 | # patch the ignore files 64 | echo "" > ota-metadata/metadata/ignore.txt; \ 65 | # build the OTA image 66 | python3 ota-metadata/metadata/ota_metadata/metadata_gen.py \ 67 | --target-dir data \ 68 | --compressed-dir data.zst \ 69 | --ignore-file ota-metadata/metadata/ignore.txt; \ 70 | python3 ota-metadata/metadata/ota_metadata/metadata_sign.py \ 71 | --sign-key "${CERTS_DIR}"/sign.key \ 72 | --cert-file sign.pem \ 73 | --persistent-file ota-metadata/metadata/persistents.txt \ 74 | --rootfs-directory data \ 75 | --compressed-rootfs-directory data.zst; \ 76 | cp ota-metadata/metadata/persistents.txt .; \ 77 | # cleanup 78 | apt-get clean; \ 79 | rm -rf \ 80 | /certs/*.key \ 81 | /tmp/* \ 82 | /var/lib/apt/lists/* \ 83 | /var/tmp/* \ 84 | ota-metadata 85 | 86 | # 87 | # ------ stage 3: output OTA image ------ # 88 | # 89 | 90 | # NOTE: use busybox so that the overall image size will not increase much, 91 | # while enable us to examine the built OTA image. 92 | 93 | FROM ${BUSYBOX_VER} 94 | 95 | ARG SYS_IMG 96 | 97 | LABEL org.opencontainers.image.description=\ 98 | "A mini OTA image for OTAClient test, based on system image ${SYS_IMG}" 99 | 100 | COPY --from=ota_img_builder /ota-image /ota-image 101 | COPY --from=ota_img_builder /certs /certs 102 | -------------------------------------------------------------------------------- /src/ota_proxy/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | import argparse 19 | import logging 20 | 21 | from . import run_otaproxy 22 | from .config import config as cfg 23 | 24 | logger = logging.getLogger("ota_proxy") 25 | 26 | if __name__ == "__main__": 27 | parser = argparse.ArgumentParser( 28 | prog="ota_proxy", 29 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 30 | description="ota_proxy server with local cache feature", 31 | ) 32 | parser.add_argument("--host", help="server listen ip", default="0.0.0.0") 33 | parser.add_argument("--port", help="server listen port", default=8080, type=int) 34 | parser.add_argument( 35 | "--upper-proxy", 36 | help="upper proxy that used for requesting remote", 37 | default="", 38 | ) 39 | parser.add_argument( 40 | "--enable-cache", 41 | help="enable local ota cache", 42 | action="store_true", 43 | default=False, 44 | ) 45 | parser.add_argument( 46 | "--enable-https", 47 | help="enable HTTPS when retrieving data from remote", 48 | action="store_true", 49 | default=False, 50 | ) 51 | parser.add_argument( 52 | "--init-cache", 53 | help="cleanup remaining cache if any", 54 | action="store_true", 55 | default=False, 56 | ) 57 | parser.add_argument( 58 | "--cache-dir", 59 | help="where to store the cache entries", 60 | default=cfg.BASE_DIR, 61 | ) 62 | parser.add_argument( 63 | "--cache-db-file", 64 | help="the location of cache db sqlite file", 65 | default=cfg.DB_FILE, 66 | ) 67 | parser.add_argument( 68 | "--external-cache-mnt-point", 69 | help=( 70 | "if specified, otaproxy will try to detect external cache dev, " 71 | "mount the dev on this mount point, and use the cache store in it." 72 | ), 73 | default=None, 74 | ) 75 | 76 | parser.add_argument( 77 | "--external-nfs-cache-mnt-point", 78 | help=( 79 | "if specified, otaproxy will try to use external NFS cache at this mount point." 80 | ), 81 | default=None, 82 | ) 83 | 84 | args = parser.parse_args() 85 | 86 | # suppress logging from third-party deps 87 | logging.basicConfig(level=logging.CRITICAL) 88 | logger.setLevel(logging.INFO) 89 | logger.info(f"launch ota_proxy at {args.host}:{args.port}") 90 | run_otaproxy( 91 | host=args.host, 92 | port=args.port, 93 | cache_dir=args.cache_dir, 94 | cache_db_f=args.cache_db_file, 95 | enable_cache=args.enable_cache, 96 | upper_proxy=args.upper_proxy, 97 | enable_https=args.enable_https, 98 | init_cache=args.init_cache, 99 | external_cache_mnt_point=args.external_cache_mnt_point, 100 | external_nfs_cache_mnt_point=args.external_nfs_cache_mnt_point, 101 | ) 102 | -------------------------------------------------------------------------------- /src/otaclient/boot_control/configs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import annotations 17 | 18 | from otaclient.configs import BootloaderType 19 | 20 | 21 | class GrubControlConfig: 22 | """x86-64 platform, with grub as bootloader.""" 23 | 24 | BOOTLOADER = BootloaderType.GRUB 25 | FSTAB_FILE_PATH = "/etc/fstab" 26 | GRUB_DIR = "/boot/grub" 27 | GRUB_CFG_FNAME = "grub.cfg" 28 | GRUB_CFG_PATH = "/boot/grub/grub.cfg" 29 | DEFAULT_GRUB_PATH = "/etc/default/grub" 30 | BOOT_OTA_PARTITION_FILE = "ota-partition" 31 | 32 | 33 | class JetsonBootCommon: 34 | # ota_status related 35 | OTA_STATUS_DIR = "/boot/ota-status" 36 | FIRMWARE_BSP_VERSION_FNAME = "firmware_bsp_version" 37 | 38 | # boot control related 39 | EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" 40 | MODEL_FPATH = "/proc/device-tree/model" 41 | NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" 42 | SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" 43 | 44 | # boot device related 45 | MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc 46 | NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd 47 | SDX_DEV_PREFIX = "sd" # non-specific device name 48 | INTERNAL_EMMC_DEVNAME = "mmcblk0" 49 | 50 | # firmware update related 51 | NVBOOTCTRL_CONF_FPATH = "/etc/nv_boot_control.conf" 52 | FIRMWARE_DPATH = "/opt/ota/firmware" 53 | FIRMWARE_UPDATE_REQUEST_FPATH = f"{FIRMWARE_DPATH}/firmware_update.yaml" 54 | FIRMWARE_MANIFEST_FPATH = f"{FIRMWARE_DPATH}/firmware_manifest.yaml" 55 | 56 | 57 | class JetsonCBootControlConfig(JetsonBootCommon): 58 | """Jetson device booted with cboot. 59 | 60 | Suuports BSP version < R34. 61 | """ 62 | 63 | BOOTLOADER = BootloaderType.JETSON_CBOOT 64 | # this path only exists on xavier 65 | TEGRA_CHIP_ID_PATH = "/sys/module/tegra_fuse/parameters/tegra_chip_id" 66 | FIRMWARE_LIST = ["bl_only_payload", "xusb_only_payload"] 67 | 68 | 69 | class JetsonUEFIBootControlConfig(JetsonBootCommon): 70 | BOOTLOADER = BootloaderType.JETSON_UEFI 71 | TEGRA_COMPAT_PATH = "/sys/firmware/devicetree/base/compatible" 72 | L4TLAUNCHER_FNAME = "BOOTAA64.efi" 73 | ESP_MOUNTPOINT = "/mnt/esp" 74 | ESP_PARTLABEL = "esp" 75 | UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" 76 | MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" 77 | CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" 78 | L4TLAUNCHER_VER_FNAME = "l4tlauncher_version" 79 | 80 | 81 | class RPIBootControlConfig: 82 | BOOTLOADER = BootloaderType.RPI_BOOT 83 | RPI_MODEL_FILE = "/proc/device-tree/model" 84 | RPI_MODEL_HINT = "Raspberry Pi 4 Model B" 85 | 86 | # boot folders 87 | SYSTEM_BOOT_MOUNT_POINT = "/boot/firmware" 88 | OTA_STATUS_DIR = "/boot/ota-status" 89 | SWITCH_BOOT_FLAG_FILE = "._ota_switch_boot_finalized" 90 | 91 | 92 | grub_cfg = GrubControlConfig() 93 | 94 | jetson_common_cfg = JetsonBootCommon() 95 | cboot_cfg = JetsonCBootControlConfig() 96 | jetson_uefi_cfg = JetsonUEFIBootControlConfig() 97 | 98 | rpi_boot_cfg = RPIBootControlConfig() 99 | -------------------------------------------------------------------------------- /tests/test_ota_metadata/test_legacy2/test_e2e.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Test OTA metadata loading with OTA image within the test container.""" 15 | 16 | from __future__ import annotations 17 | 18 | import logging 19 | import os 20 | from hashlib import sha256 21 | from pathlib import Path 22 | from queue import Queue 23 | 24 | from pytest_mock import MockerFixture 25 | 26 | from ota_metadata.legacy2.metadata import MetadataJWTParser 27 | from ota_metadata.utils.cert_store import load_ca_cert_chains 28 | from otaclient._types import MultipleECUStatusFlags 29 | from otaclient._utils import SharedOTAClientMetricsReader 30 | from otaclient.metrics import OTAMetricsData 31 | from otaclient.ota_core._updater_base import LegacyOTAImageSupportMixin 32 | from otaclient_common.downloader import DownloaderPool 33 | from tests.conftest import ( 34 | CERTS_DIR, 35 | OTA_IMAGE_DIR, 36 | OTA_IMAGE_SIGN_CERT, 37 | cfg, 38 | ) 39 | from tests.test_ota_metadata.conftest import iter_helper 40 | 41 | METADATA_JWT = OTA_IMAGE_DIR / "metadata.jwt" 42 | 43 | logger = logging.getLogger(__name__) 44 | 45 | 46 | def test_metadata_jwt_parser_e2e() -> None: 47 | metadata_jwt = METADATA_JWT.read_text() 48 | sign_cert = OTA_IMAGE_SIGN_CERT.read_bytes() 49 | ca_store = load_ca_cert_chains(CERTS_DIR) 50 | 51 | parser = MetadataJWTParser( 52 | metadata_jwt, 53 | ca_chains_store=ca_store, 54 | ) 55 | 56 | # step1: verify sign cert against CA store 57 | parser.verify_metadata_cert(sign_cert) 58 | # step2: verify metadata.jwt against sign cert 59 | parser.verify_metadata_signature(sign_cert) 60 | 61 | 62 | def test_download_and_parse_metadata(tmp_path: Path, mocker: MockerFixture): 63 | legacy_ota_image = LegacyOTAImageSupportMixin( 64 | version="dummy_version", 65 | raw_url_base=cfg.OTA_IMAGE_URL, 66 | session_wd=tmp_path, 67 | downloader_pool=DownloaderPool(instance_num=3, hash_func=sha256), 68 | session_id=f"session_id_{os.urandom(2).hex()}", 69 | ecu_status_flags=mocker.MagicMock(spec=MultipleECUStatusFlags), 70 | status_report_queue=mocker.MagicMock(spec=Queue), 71 | metrics=mocker.MagicMock(spec=OTAMetricsData), 72 | shm_metrics_reader=mocker.MagicMock(spec=SharedOTAClientMetricsReader), 73 | ) # type: ignore 74 | 75 | ca_chains_store = load_ca_cert_chains(cfg.CERTS_DIR) 76 | legacy_ota_image.setup_ota_image_support(ca_chains_store=ca_chains_store) 77 | legacy_ota_image._process_metadata() 78 | 79 | # ------ check result ------ # 80 | ota_metadata = legacy_ota_image._ota_metadata 81 | fst_helper = ota_metadata.file_table_helper 82 | assert iter_helper(fst_helper.iter_dir_entries()) == ota_metadata.total_dirs_num 83 | # NOTE: for legacy OTA image, non_regular_files catagory only has symlink 84 | assert ( 85 | iter_helper(fst_helper.iter_non_regular_entries()) 86 | == ota_metadata.total_symlinks_num 87 | ) 88 | assert ( 89 | iter_helper(fst_helper.iter_regular_entries()) 90 | == ota_metadata.total_regulars_num 91 | ) 92 | -------------------------------------------------------------------------------- /src/otaclient/create_standby/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | import logging 18 | import os 19 | from pathlib import Path 20 | from typing import Generator 21 | 22 | from otaclient_common import human_readable_size 23 | from otaclient_common._typing import StrOrPath 24 | from otaclient_common.cmdhelper import ( 25 | ensure_mount, 26 | ensure_umount, 27 | get_attrs_by_dev, 28 | mount_ro, 29 | ) 30 | from otaclient_common.linux import subprocess_run_wrapper 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | class TopDownCommonShortestPath: 36 | """Assume that the disk scan is top-down style.""" 37 | 38 | def __init__(self) -> None: 39 | self._store: set[Path] = set() 40 | 41 | def add_path(self, _path: Path): 42 | for _parent in _path.parents: 43 | # this path is covered by a shorter common prefix 44 | if _parent in self._store: 45 | return 46 | self._store.add(_path) 47 | 48 | def iter_paths(self) -> Generator[Path]: 49 | yield from self._store 50 | 51 | 52 | def _check_if_ext4(dev: StrOrPath) -> bool: # pragma: no cover 53 | return get_attrs_by_dev("FSTYPE", dev) == "ext4" 54 | 55 | 56 | def _check_if_fs_healthy(dev: StrOrPath) -> bool: # pragma: no cover 57 | _res = subprocess_run_wrapper( 58 | ["e2fsck", "-n", str(dev)], 59 | check=False, 60 | check_output=False, 61 | ) 62 | return _res.returncode == 0 63 | 64 | 65 | def _get_fs_used_size(mnt_point: StrOrPath) -> int: # pragma: no cover 66 | stats = os.statvfs(mnt_point) 67 | used_in_bytes = (stats.f_blocks - stats.f_bfree) * stats.f_frsize 68 | logger.info(f"fs on {mnt_point=} used size: {human_readable_size(used_in_bytes)}") 69 | return used_in_bytes 70 | 71 | 72 | def _check_fs_used_size_reach_threshold( 73 | dev: StrOrPath, mnt_point: StrOrPath, threshold_in_bytes: int 74 | ) -> bool: # pragma: no cover 75 | try: 76 | ensure_mount(dev, mnt_point, mount_func=mount_ro, raise_exception=True) 77 | return _get_fs_used_size(mnt_point) >= threshold_in_bytes 78 | except Exception as e: 79 | logger.warning(f"failed to mount standby slot ({dev=}: {e!r}") 80 | return False 81 | finally: 82 | ensure_umount(mnt_point, ignore_error=True) 83 | 84 | 85 | def can_use_in_place_mode( 86 | dev: StrOrPath, mnt_point: StrOrPath, threshold_in_bytes: int | None = None 87 | ) -> bool: # pragma: no cover 88 | """ 89 | Check whether target standby slot device is ready for in-place update mode. 90 | 91 | The following checks will be performed in order: 92 | 1. standby slot contains ext4 partition. 93 | 2. standby slot's fs is healthy. 94 | 3. standby slot's used size reaches threshold. 95 | """ 96 | if not (_check_if_ext4(dev) and _check_if_fs_healthy(dev)): 97 | return False 98 | if threshold_in_bytes is not None: 99 | return _check_fs_used_size_reach_threshold(dev, mnt_point, threshold_in_bytes) 100 | return True 101 | -------------------------------------------------------------------------------- /proto/hatch_build.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Dynamically generate protobuf python code.""" 15 | 16 | 17 | from __future__ import annotations 18 | 19 | import os 20 | import shutil 21 | import subprocess 22 | import sys 23 | from pathlib import Path 24 | from typing import Any 25 | 26 | from hatchling.builders.hooks.plugin.interface import BuildHookInterface 27 | 28 | OUTPUT_BASE = "src" 29 | 30 | 31 | def _protoc_compile( 32 | proto_file: str, 33 | output_base: str, 34 | output_package: str, 35 | *, 36 | extra_imports: list[str] | None, 37 | work_dir: str | Path, 38 | ): 39 | # copy the proto_file to the output_dir to retain the proper package import layout 40 | output_package_dir = Path(output_base) / output_package 41 | shutil.copy(proto_file, output_package_dir) 42 | _proto_file = output_package_dir / os.path.basename(proto_file) 43 | 44 | # fmt: off 45 | cmd = [ 46 | sys.executable, "-m", "grpc_tools.protoc", 47 | f"--python_out={output_base}", 48 | f"--pyi_out={output_base}", 49 | f"--grpc_python_out={output_base}", 50 | ] 51 | # fmt: on 52 | if isinstance(extra_imports, list): 53 | for _import in extra_imports: 54 | cmd.append(f"-I{_import}") 55 | # also place the output_base as import path 56 | cmd.append(f"-I{output_base}") 57 | 58 | cmd.append(str(_proto_file)) 59 | print(f"call protoc with: {' '.join(cmd)}") 60 | subprocess.check_call(cmd, cwd=work_dir) 61 | 62 | 63 | class CustomBuildHook(BuildHookInterface): 64 | """ 65 | Configs: 66 | api_version: the API version of otaclient service API. 67 | """ 68 | 69 | def initialize(self, version: str, build_data: dict[str, Any]) -> None: 70 | for config in self.config["proto_builds"]: 71 | # ------ parse config ------ # 72 | proto_file = config["proto_file"] 73 | print(f"build protoc python code for {proto_file} ...") 74 | 75 | output_package = config["output_package"] 76 | extra_imports = config.get("extra_imports", []) 77 | 78 | # ------ load consts ------ # 79 | output_base = OUTPUT_BASE 80 | 81 | # ------ validate config ------ # 82 | if not isinstance(extra_imports, list): 83 | raise ValueError( 84 | f"expect extra_imports to be a list, get {type(extra_imports)}" 85 | ) 86 | 87 | if not os.path.exists(proto_file): 88 | raise FileNotFoundError(f"{proto_file=} not found, abort") 89 | 90 | # let the folder where proto file exists become the work_dir 91 | work_dir = os.path.dirname(proto_file) 92 | if not work_dir: 93 | work_dir = "." 94 | 95 | # ------ compile proto file ------ # 96 | _protoc_compile( 97 | proto_file, 98 | extra_imports=extra_imports, 99 | output_base=output_base, 100 | output_package=output_package, 101 | work_dir=work_dir, 102 | ) 103 | -------------------------------------------------------------------------------- /tests/test_ota_proxy/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pytest 4 | 5 | from ota_proxy.utils import process_raw_url 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "_input, _enable_https, _expected", 10 | ( 11 | # Basic ASCII path 12 | ("http://example.com/usr/local/bin", False, "http://example.com/usr/local/bin"), 13 | ("http://example.com/usr/local/bin", True, "https://example.com/usr/local/bin"), 14 | # Path with space 15 | ( 16 | "http://example.com/home/user/My Documents", 17 | False, 18 | "http://example.com/home/user/My%20Documents", 19 | ), 20 | ( 21 | "http://example.com/home/user/My Documents", 22 | True, 23 | "https://example.com/home/user/My%20Documents", 24 | ), 25 | # Unicode path 26 | ( 27 | "http://example.com/home/user/Café Bébé", 28 | False, 29 | "http://example.com/home/user/Caf%C3%A9%20B%C3%A9b%C3%A9", 30 | ), 31 | ( 32 | "http://example.com/home/user/Café Bébé", 33 | True, 34 | "https://example.com/home/user/Caf%C3%A9%20B%C3%A9b%C3%A9", 35 | ), 36 | # Special shell characters 37 | ( 38 | "http://example.com/tmp/a&b|c>output", 39 | False, 40 | "http://example.com/tmp/a%26b%7Cc%3Eoutput", 41 | ), 42 | ( 43 | "http://example.com/tmp/a&b|c>output", 44 | True, 45 | "https://example.com/tmp/a%26b%7Cc%3Eoutput", 46 | ), 47 | # Question mark and fragment as literal path part (not query/fragment) 48 | ( 49 | "http://example.com/data/some?file#name", 50 | False, 51 | "http://example.com/data/some%3Ffile%23name", 52 | ), 53 | ( 54 | "http://example.com/data/some?file#name", 55 | True, 56 | "https://example.com/data/some%3Ffile%23name", 57 | ), 58 | # File name with plus and percent signs 59 | ( 60 | "http://example.com/home/user/file+name%.txt", 61 | False, 62 | "http://example.com/home/user/file%2Bname%25.txt", 63 | ), 64 | ( 65 | "http://example.com/home/user/file+name%.txt", 66 | True, 67 | "https://example.com/home/user/file%2Bname%25.txt", 68 | ), 69 | # Dotfiles and hidden folders 70 | ( 71 | "http://example.com/home/user/.cache/.myconfig", 72 | False, 73 | "http://example.com/home/user/.cache/.myconfig", 74 | ), 75 | ( 76 | "http://example.com/home/user/.cache/.myconfig", 77 | True, 78 | "https://example.com/home/user/.cache/.myconfig", 79 | ), 80 | # Emoji in filename 81 | ( 82 | "http://example.com/home/user/📁.txt", 83 | False, 84 | "http://example.com/home/user/%F0%9F%93%81.txt", 85 | ), 86 | ( 87 | "http://example.com/home/user/📁.txt", 88 | True, 89 | "https://example.com/home/user/%F0%9F%93%81.txt", 90 | ), 91 | # Trailing space and newline 92 | ( 93 | "http://example.com/home/user/file \n", 94 | False, 95 | "http://example.com/home/user/file%20%0A", 96 | ), 97 | ( 98 | "http://example.com/home/user/file \n", 99 | True, 100 | "https://example.com/home/user/file%20%0A", 101 | ), 102 | ), 103 | ) 104 | def test_process_raw_url(_input: str, _enable_https: bool, _expected: str): 105 | assert process_raw_url(_input, _enable_https) == _expected 106 | -------------------------------------------------------------------------------- /tests/test_otaclient_common/test_proto_wrapper/example_pb2.pyi: -------------------------------------------------------------------------------- 1 | from typing import ClassVar as _ClassVar 2 | from typing import Iterable as _Iterable 3 | from typing import Mapping as _Mapping 4 | from typing import Optional as _Optional 5 | from typing import Union as _Union 6 | 7 | from google.protobuf import descriptor as _descriptor 8 | from google.protobuf import duration_pb2 as _duration_pb2 9 | from google.protobuf import message as _message 10 | from google.protobuf.internal import containers as _containers 11 | from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper 12 | 13 | DESCRIPTOR: _descriptor.FileDescriptor 14 | VALUE_0: SampleEnum 15 | VALUE_1: SampleEnum 16 | VALUE_2: SampleEnum 17 | 18 | class InnerMessage(_message.Message): 19 | __slots__ = [ 20 | "double_field", 21 | "duration_field", 22 | "enum_field", 23 | "int_field", 24 | "str_field", 25 | ] 26 | DOUBLE_FIELD_FIELD_NUMBER: _ClassVar[int] 27 | DURATION_FIELD_FIELD_NUMBER: _ClassVar[int] 28 | ENUM_FIELD_FIELD_NUMBER: _ClassVar[int] 29 | INT_FIELD_FIELD_NUMBER: _ClassVar[int] 30 | STR_FIELD_FIELD_NUMBER: _ClassVar[int] 31 | double_field: float 32 | duration_field: _duration_pb2.Duration 33 | enum_field: SampleEnum 34 | int_field: int 35 | str_field: str 36 | def __init__( 37 | self, 38 | int_field: _Optional[int] = ..., 39 | double_field: _Optional[float] = ..., 40 | str_field: _Optional[str] = ..., 41 | duration_field: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., 42 | enum_field: _Optional[_Union[SampleEnum, str]] = ..., 43 | ) -> None: ... 44 | 45 | class OuterMessage(_message.Message): 46 | __slots__ = [ 47 | "mapping_composite_field", 48 | "mapping_scalar_field", 49 | "nested_msg", 50 | "repeated_composite_field", 51 | "repeated_scalar_field", 52 | ] 53 | 54 | class MappingCompositeFieldEntry(_message.Message): 55 | __slots__ = ["key", "value"] 56 | KEY_FIELD_NUMBER: _ClassVar[int] 57 | VALUE_FIELD_NUMBER: _ClassVar[int] 58 | key: int 59 | value: InnerMessage 60 | def __init__( 61 | self, 62 | key: _Optional[int] = ..., 63 | value: _Optional[_Union[InnerMessage, _Mapping]] = ..., 64 | ) -> None: ... 65 | 66 | class MappingScalarFieldEntry(_message.Message): 67 | __slots__ = ["key", "value"] 68 | KEY_FIELD_NUMBER: _ClassVar[int] 69 | VALUE_FIELD_NUMBER: _ClassVar[int] 70 | key: str 71 | value: str 72 | def __init__( 73 | self, key: _Optional[str] = ..., value: _Optional[str] = ... 74 | ) -> None: ... 75 | 76 | MAPPING_COMPOSITE_FIELD_FIELD_NUMBER: _ClassVar[int] 77 | MAPPING_SCALAR_FIELD_FIELD_NUMBER: _ClassVar[int] 78 | NESTED_MSG_FIELD_NUMBER: _ClassVar[int] 79 | REPEATED_COMPOSITE_FIELD_FIELD_NUMBER: _ClassVar[int] 80 | REPEATED_SCALAR_FIELD_FIELD_NUMBER: _ClassVar[int] 81 | mapping_composite_field: _containers.MessageMap[int, InnerMessage] 82 | mapping_scalar_field: _containers.ScalarMap[str, str] 83 | nested_msg: InnerMessage 84 | repeated_composite_field: _containers.RepeatedCompositeFieldContainer[InnerMessage] 85 | repeated_scalar_field: _containers.RepeatedScalarFieldContainer[str] 86 | def __init__( 87 | self, 88 | repeated_scalar_field: _Optional[_Iterable[str]] = ..., 89 | repeated_composite_field: _Optional[ 90 | _Iterable[_Union[InnerMessage, _Mapping]] 91 | ] = ..., 92 | nested_msg: _Optional[_Union[InnerMessage, _Mapping]] = ..., 93 | mapping_scalar_field: _Optional[_Mapping[str, str]] = ..., 94 | mapping_composite_field: _Optional[_Mapping[int, InnerMessage]] = ..., 95 | ) -> None: ... 96 | 97 | class SampleEnum(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): 98 | __slots__ = [] 99 | -------------------------------------------------------------------------------- /src/otaclient/grpc/api_v2/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 TIER IV, INC. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Main entry for OTA API v2 grpc server.""" 15 | 16 | 17 | from __future__ import annotations 18 | 19 | import asyncio 20 | import atexit 21 | import logging 22 | from concurrent.futures import ThreadPoolExecutor 23 | from multiprocessing.queues import Queue as mp_Queue 24 | from typing import Callable, NoReturn 25 | 26 | from otaclient._types import ( 27 | CriticalZoneFlag, 28 | IPCRequest, 29 | IPCResponse, 30 | MultipleECUStatusFlags, 31 | StopOTAFlag, 32 | ) 33 | from otaclient._utils import SharedOTAClientStatusReader 34 | 35 | logger = logging.getLogger(__name__) 36 | 37 | 38 | def grpc_server_process( 39 | *, 40 | shm_reader_factory: Callable[[], SharedOTAClientStatusReader], 41 | op_queue: mp_Queue[IPCRequest], 42 | resp_queue: mp_Queue[IPCResponse], 43 | ecu_status_flags: MultipleECUStatusFlags, 44 | critical_zone_flag: CriticalZoneFlag, 45 | stop_ota_flag: StopOTAFlag, 46 | ) -> NoReturn: # type: ignore 47 | from otaclient._logging import configure_logging 48 | 49 | configure_logging() 50 | logger.info("otaclient OTA API grpc server started") 51 | 52 | shm_reader = shm_reader_factory() 53 | atexit.register(shm_reader.atexit) 54 | 55 | async def _grpc_server_launcher(): 56 | import grpc.aio 57 | 58 | from otaclient.configs.cfg import cfg, ecu_info 59 | from otaclient.grpc.api_v2.ecu_status import ECUStatusStorage 60 | from otaclient.grpc.api_v2.ecu_tracker import ECUTracker 61 | from otaclient.grpc.api_v2.servicer import OTAClientAPIServicer 62 | from otaclient_api.v2 import otaclient_v2_pb2_grpc as v2_grpc 63 | from otaclient_api.v2.api_stub import OtaClientServiceV2 64 | 65 | ecu_status_storage = ECUStatusStorage(ecu_status_flags=ecu_status_flags) 66 | ecu_tracker = ECUTracker(ecu_status_storage, shm_reader) 67 | ecu_tracker.start() 68 | 69 | thread_pool = ThreadPoolExecutor( 70 | thread_name_prefix="ota_api_server", 71 | ) 72 | api_servicer = OTAClientAPIServicer( 73 | ecu_status_storage=ecu_status_storage, 74 | op_queue=op_queue, 75 | resp_queue=resp_queue, 76 | critical_zone_flag=critical_zone_flag, 77 | stop_ota_flag=stop_ota_flag, 78 | executor=thread_pool, 79 | ) 80 | ota_client_service_v2 = OtaClientServiceV2(api_servicer) 81 | 82 | server = grpc.aio.server(migration_thread_pool=thread_pool) 83 | v2_grpc.add_OtaClientServiceServicer_to_server( 84 | server=server, servicer=ota_client_service_v2 85 | ) 86 | _address_info = f"{ecu_info.ip_addr}:{cfg.OTA_API_SERVER_PORT}" 87 | server.add_insecure_port(_address_info) 88 | 89 | try: 90 | logger.info(f"launch grpc API server at {_address_info}") 91 | await server.start() 92 | logger.info("gRPC API server started successfully") 93 | await server.wait_for_termination() 94 | except Exception as e: 95 | logger.exception(f"gRPC server terminated with exception: {e}") 96 | finally: 97 | await server.stop(1) 98 | thread_pool.shutdown(wait=True) 99 | 100 | asyncio.run(_grpc_server_launcher()) 101 | --------------------------------------------------------------------------------