├── .github └── workflows │ ├── docker-build.yml │ └── tests.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.rst ├── media ├── CustomPiOS.png ├── CustomPiOS.svg └── rpi-imager-CustomPiOS.png ├── pyproject.toml ├── src ├── Dockerfile ├── argparse.bash ├── base_image_downloader_wrapper.sh ├── build ├── build_custom_os ├── build_docker ├── chroot_in_to_image.sh ├── common.sh ├── config ├── config_sanity ├── custompios ├── custompios_core │ ├── __init__.py │ ├── base_image_downloader.py │ ├── common.py │ ├── execution_order.py │ ├── generate_board_config.py │ ├── get_remote_modules.py │ ├── list_boards.py │ ├── make_rpi-imager-snipplet.py │ ├── multi-arch-manifest.yaml │ └── multi_build.py ├── dist_generators │ ├── dist_example │ │ └── src │ │ │ ├── .gitignore │ │ │ ├── build_dist │ │ │ ├── config │ │ │ ├── image │ │ │ ├── .gitignore │ │ │ └── README │ │ │ ├── modules │ │ │ └── example │ │ │ │ ├── config │ │ │ │ ├── end_chroot_script │ │ │ │ ├── filesystem │ │ │ │ ├── boot │ │ │ │ │ └── README │ │ │ │ ├── home │ │ │ │ │ ├── pi │ │ │ │ │ │ └── README │ │ │ │ │ └── root │ │ │ │ │ │ └── README │ │ │ │ └── root │ │ │ │ │ └── README │ │ │ │ └── start_chroot_script │ │ │ └── vagrant │ │ │ ├── Vagrantfile │ │ │ ├── run_vagrant_build.sh │ │ │ └── setup.sh │ └── dist_example_script ├── docker-compose.yml ├── docker │ └── docker-compose.yml ├── hooks │ ├── post_push │ └── pre_build ├── images.yml ├── make_custom_pi_os ├── misc │ └── jenkins-ci │ │ └── console_parsing ├── modules │ ├── admin-toolkit │ │ ├── config │ │ ├── filesystem │ │ │ ├── home │ │ │ │ └── pi │ │ │ │ │ └── scripts │ │ │ │ │ └── ufw_config │ │ │ ├── root_init │ │ │ │ └── etc │ │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ └── ufw_config.service │ │ │ └── tools │ │ │ │ ├── FullPageHdmiScripts │ │ │ │ ├── tv_off.sh │ │ │ │ └── tv_on.sh │ │ │ │ ├── HostNameScript │ │ │ │ └── hostname_change.sh │ │ │ │ └── cronJobs │ │ │ │ ├── User │ │ │ │ └── pi │ │ │ │ └── system │ │ │ │ └── crontab │ │ └── start_chroot_script │ ├── auto-hotspot │ │ ├── config │ │ ├── filesystem │ │ │ └── root │ │ │ │ ├── etc │ │ │ │ ├── dhcp │ │ │ │ │ └── dhclient-exit-hooks.d │ │ │ │ │ │ └── prefix_delegation │ │ │ │ ├── hostapd │ │ │ │ │ └── hostapd.conf │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ └── autohotspot.service │ │ │ │ └── usr │ │ │ │ └── bin │ │ │ │ ├── add_masquerade │ │ │ │ ├── autohotspotN │ │ │ │ └── make_sure_master │ │ └── start_chroot_script │ ├── auto-mount-removable │ │ ├── filesystem │ │ │ └── root │ │ │ │ └── etc │ │ │ │ └── systemd │ │ │ │ └── system │ │ │ │ └── systemd-udevd.service.d │ │ │ │ └── override-private.conf │ │ └── start_chroot_script │ ├── base │ │ ├── config │ │ ├── end_chroot_script │ │ ├── filesystem │ │ │ └── ubuntu │ │ │ │ └── usr │ │ │ │ └── lib │ │ │ │ └── dhcpcd │ │ │ │ └── dhcpcd-hooks │ │ │ │ └── 10-wpa_supplicant │ │ ├── meta │ │ └── start_chroot_script │ ├── cockpit-install │ │ ├── config │ │ ├── filesystem │ │ │ ├── home │ │ │ │ └── pi │ │ │ │ │ └── scripts │ │ │ │ │ └── install_cockpit │ │ │ └── root_init │ │ │ │ └── etc │ │ │ │ └── systemd │ │ │ │ └── system │ │ │ │ └── cockpit_installer.service │ │ └── start_chroot_script │ ├── disable-services │ │ ├── end_chroot_script │ │ └── start_chroot_script │ ├── docker │ │ ├── config │ │ ├── filesystem │ │ │ ├── boot │ │ │ │ └── docker-compose │ │ │ │ │ └── README │ │ │ └── root │ │ │ │ ├── etc │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ └── docker-compose.service │ │ │ │ └── usr │ │ │ │ └── bin │ │ │ │ ├── start_docker_compose │ │ │ │ └── stop_docker_compose │ │ └── start_chroot_script │ ├── ffmpeg │ │ ├── config │ │ ├── end_chroot_script │ │ └── start_chroot_script │ ├── gui │ │ ├── config │ │ ├── end_chroot_script │ │ ├── filesystem │ │ │ ├── opt │ │ │ │ └── scripts │ │ │ │ │ ├── enable_gpu │ │ │ │ │ ├── rotate.sh │ │ │ │ │ ├── start_gui │ │ │ │ │ └── update_lightdm_conf │ │ │ └── root_init │ │ │ │ ├── etc │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ ├── enable_gpu_first_boot.service │ │ │ │ │ └── update_lightdm_conf.service │ │ │ │ └── usr │ │ │ │ └── share │ │ │ │ └── xsessions │ │ │ │ └── guisession.desktop │ │ └── start_chroot_script │ ├── install-linux-modules-extra │ │ ├── filesystem │ │ │ └── root │ │ │ │ └── fix-flash-kernel-rpi4.patch │ │ └── start_chroot_script │ ├── kernel │ │ ├── config │ │ ├── end_chroot_script │ │ └── start_chroot_script │ ├── mysql │ │ ├── config │ │ └── start_chroot_script │ ├── network │ │ ├── config │ │ ├── filesystem │ │ │ ├── etc │ │ │ │ ├── systemd │ │ │ │ │ └── system │ │ │ │ │ │ └── disable-wifi-pwr-mgmt.service │ │ │ │ └── udev │ │ │ │ │ └── rules.d │ │ │ │ │ └── 070-wifi-powersave.rules │ │ │ ├── network-manager │ │ │ │ ├── boot │ │ │ │ │ └── wifi.nmconnection │ │ │ │ └── root │ │ │ │ │ ├── etc │ │ │ │ │ └── systemd │ │ │ │ │ │ └── system │ │ │ │ │ │ └── copy-network-manager-config@.service │ │ │ │ │ └── opt │ │ │ │ │ └── custompios │ │ │ │ │ └── copy-network-manager-config │ │ │ ├── usr │ │ │ │ └── local │ │ │ │ │ └── bin │ │ │ │ │ ├── pwrsave │ │ │ │ │ └── pwrsave-udev │ │ │ └── wpa-supplicant │ │ │ │ └── boot │ │ │ │ └── custompios-wpa-supplicant.txt │ │ ├── meta │ │ └── start_chroot_script │ ├── password-for-sudo │ │ └── start_chroot_script │ ├── pkgupgrade │ │ ├── config │ │ ├── end_chroot_script │ │ └── start_chroot_script │ ├── raspicam │ │ ├── config │ │ └── start_chroot_script │ ├── readonly │ │ ├── config │ │ ├── filesystem │ │ │ └── boot │ │ │ │ └── cmdline.txt │ │ └── start_chroot_script │ ├── usage-statistics │ │ ├── config │ │ ├── filesystem │ │ │ ├── root │ │ │ │ ├── etc │ │ │ │ │ └── systemd │ │ │ │ │ │ └── system │ │ │ │ │ │ └── usage-statistics.service │ │ │ │ └── usr │ │ │ │ │ └── bin │ │ │ │ │ └── boot_report │ │ │ └── root_lepotato │ │ │ │ └── usr │ │ │ │ └── bin │ │ │ │ └── boot_report │ │ └── start_chroot_script │ └── usbconsole │ │ ├── config │ │ └── start_chroot_script ├── modules_remote.yml ├── nightly_build_scripts │ └── custompios_nightly_build ├── qemu_boot.sh ├── qemu_boot64.sh ├── release ├── requirements-devel.txt ├── requirements.txt ├── update-custompios-paths └── variants │ ├── armbian │ ├── config │ ├── filesystem │ │ └── root │ │ │ └── etc │ │ │ ├── network │ │ │ └── interfaces │ │ │ └── wpa_supplicant │ │ │ └── wpa_supplicant.conf │ ├── post_chroot_script │ └── pre_chroot_script │ ├── bananapi-m1 │ ├── config │ ├── filesystem │ │ └── root │ │ │ └── etc │ │ │ └── resolv.conf │ ├── post_chroot_script │ └── pre_chroot_script │ ├── example │ ├── config │ ├── config.nightly │ ├── filesystem │ │ └── root │ │ │ └── etc │ │ │ └── dhclient.conf │ └── post_chroot_script │ ├── jessielite │ └── config │ └── raspios_lite_arm64 │ └── config └── tests └── test_qemu_setup.sh /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | push: 4 | branches: 5 | - "master" 6 | - "devel" 7 | - "docker-github-actions" 8 | - "release/v1" 9 | - "beta" 10 | tags: 11 | - "*" 12 | jobs: 13 | docker: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - 17 | name: 📥 Checkout 18 | uses: actions/checkout@v4 19 | - 20 | name: 🖥️ Set up QEMU 21 | uses: docker/setup-qemu-action@v3 22 | - 23 | name: 🏷️ Docker meta tag 24 | id: meta 25 | uses: docker/metadata-action@v5 26 | with: 27 | images: | 28 | ghcr.io/${{ github.repository_owner }}/custompios 29 | tags: | 30 | type=ref,event=branch 31 | type=ref,event=tag 32 | type=sha 33 | - 34 | name: 🛠️ Set up Docker Buildx 35 | uses: docker/setup-buildx-action@v3 36 | - 37 | name: 🔑 Login to GitHub Container Registry 38 | uses: docker/login-action@v3 39 | with: 40 | registry: ghcr.io 41 | username: ${{ github.repository_owner }} 42 | password: ${{ secrets.GITHUB_TOKEN }} 43 | - 44 | name: 🏗️ Build and push 45 | id: docker_build 46 | uses: docker/build-push-action@v5 47 | with: 48 | context: src 49 | file: src/Dockerfile 50 | platforms: linux/amd64,linux/arm64,linux/arm/v7 51 | push: true 52 | tags: ${{ steps.meta.outputs.tags }} 53 | labels: ${{ steps.meta.outputs.labels }} 54 | secrets: | 55 | github_token=${{ secrets.GITHUB_TOKEN }} 56 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Run Tests via Makefile 2 | 3 | on: 4 | push: 5 | branches: 6 | - "master" 7 | - "devel" 8 | - "docker-github-actions" 9 | - "release/v1" 10 | - "feature/unit-testing" 11 | - "beta" 12 | tags: 13 | - "*" 14 | 15 | jobs: 16 | test: 17 | name: Run Makefile Tests 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | # - name: Install dependencies 24 | # run: | 25 | # sudo apt-get update 26 | # sudo apt-get install -y qemu-user-static binfmt-support 27 | 28 | - name: Run Makefile 29 | run: make test 30 | 31 | - name: Archive test results 32 | if: always() 33 | uses: actions/upload-artifact@v4 34 | with: 35 | name: test-results 36 | path: tests/ 37 | retention-days: 60 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | src/config.local 2 | src/image/*.zip 3 | src/image/*.xz 4 | **/key.json 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for running shell script tests 2 | 3 | # Shell to use 4 | SHELL := /bin/bash 5 | 6 | # Find all test files 7 | TEST_DIR := tests 8 | TEST_FILES := $(wildcard $(TEST_DIR)/test_*.sh) 9 | 10 | # Colors for output 11 | BLUE := \033[1;34m 12 | GREEN := \033[1;32m 13 | RED := \033[1;31m 14 | NC := \033[0m # No Color 15 | 16 | .PHONY: all test clean help 17 | 18 | # Default target 19 | all: test 20 | 21 | # Help message 22 | help: 23 | @echo "Available targets:" 24 | @echo " make test - Run all tests" 25 | @echo " make clean - Clean up temporary files" 26 | @echo " make help - Show this help message" 27 | 28 | # Run all tests 29 | test: 30 | @echo -e "$(BLUE)Running tests...$(NC)" 31 | @echo "═══════════════════════════════════════" 32 | @success=true; \ 33 | for test in $(TEST_FILES); do \ 34 | echo -e "$(BLUE)Running $$test...$(NC)"; \ 35 | if chmod +x $$test && ./$$test; then \ 36 | echo -e "$(GREEN)✓ $$test passed$(NC)"; \ 37 | else \ 38 | echo -e "$(RED)✗ $$test failed$(NC)"; \ 39 | success=false; \ 40 | fi; \ 41 | echo "───────────────────────────────────────"; \ 42 | done; \ 43 | echo ""; \ 44 | if $$success; then \ 45 | echo -e "$(GREEN)All tests passed successfully!$(NC)"; \ 46 | exit 0; \ 47 | else \ 48 | echo -e "$(RED)Some tests failed!$(NC)"; \ 49 | exit 1; \ 50 | fi 51 | 52 | # Clean up any temporary files (if needed) 53 | clean: 54 | @echo -e "$(BLUE)Cleaning up...$(NC)" 55 | @find $(TEST_DIR) -type f -name "*.tmp" -delete 56 | @find $(TEST_DIR) -type f -name "*.log" -delete 57 | @echo -e "$(GREEN)Cleanup complete$(NC)" 58 | -------------------------------------------------------------------------------- /media/CustomPiOS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guysoft/CustomPiOS/0f05cdaa6197ab0437fd286abc94b00e1899ac69/media/CustomPiOS.png -------------------------------------------------------------------------------- /media/rpi-imager-CustomPiOS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guysoft/CustomPiOS/0f05cdaa6197ab0437fd286abc94b00e1899ac69/media/rpi-imager-CustomPiOS.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "custompios" 3 | version = "2.0.0" 4 | description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship." 5 | authors = ["Guy Sheffer "] 6 | license = "GPLv3" 7 | readme = "README.rst" 8 | packages = [ 9 | # { include = "src/*" }, 10 | { include = "custompios_core", from = "src" } 11 | ] 12 | 13 | [tool.poetry.dependencies] 14 | python = "^3.11" 15 | GitPython = "^3.1.41" 16 | 17 | [tool.poetry.group.dev.dependencies] 18 | types-PyYAML = "^6.0.12.12" 19 | 20 | [tool.poetry.scripts] 21 | custompios_build = 'custompios_core.multi_build:main' 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /src/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm-backports 2 | 3 | MAINTAINER Guy Sheffer 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | ca-certificates \ 8 | sudo \ 9 | build-essential \ 10 | sudo \ 11 | curl \ 12 | git \ 13 | wget \ 14 | p7zip-full \ 15 | python3 \ 16 | python3-distutils \ 17 | python3-dev \ 18 | python3-git \ 19 | python3-yaml \ 20 | binfmt-support \ 21 | qemu-system \ 22 | qemu-user \ 23 | qemu-user-static \ 24 | sudo \ 25 | file \ 26 | fdisk \ 27 | btrfs-progs \ 28 | jq \ 29 | zip \ 30 | xz-utils \ 31 | lsof \ 32 | f2fs-tools \ 33 | && rm -rf /var/lib/apt/lists/* \ 34 | && apt -qyy clean 35 | 36 | RUN ln -s /CustomPiOS/nightly_build_scripts/custompios_nightly_build /usr/bin/build 37 | RUN ln -s /usr/bin/python3 /usr/bin/python 38 | 39 | COPY . /CustomPiOS 40 | 41 | CMD ["/bin/bash"] 42 | -------------------------------------------------------------------------------- /src/argparse.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Use python's argparse module in shell scripts 4 | # 5 | # The function `argparse` parses its arguments using 6 | # argparse.ArgumentParser; the parser is defined in the function's 7 | # stdin. 8 | # 9 | # Executing ``argparse.bash`` (as opposed to sourcing it) prints a 10 | # script template. 11 | # 12 | # https://github.com/nhoffman/argparse-bash 13 | # MIT License - Copyright (c) 2015 Noah Hoffman 14 | 15 | # Get python executable 16 | if which python > /dev/null; then 17 | PYTHON=$(which python) 18 | elif which python3 > /dev/null; then 19 | PYTHON=$(which python3) 20 | else 21 | echo "No python 3 executable found" 22 | exit 1 23 | fi 24 | 25 | argparse(){ 26 | argparser=$(mktemp 2>/dev/null || mktemp -t argparser) 27 | cat > "$argparser" <> "$argparser" 45 | 46 | cat >> "$argparser" < /dev/null; then 64 | eval $($PYTHON "$argparser" "$@") 65 | retval=0 66 | else 67 | $PYTHON "$argparser" "$@" 68 | retval=1 69 | fi 70 | 71 | rm "$argparser" 72 | return $retval 73 | } 74 | 75 | if echo $0 | grep -q argparse.bash; then 76 | cat <&1 | tee "$LOG" 32 | exit ${PIPESTATUS} 33 | else 34 | eval "$SCRIPT" 35 | fi 36 | -------------------------------------------------------------------------------- /src/build_custom_os: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | 5 | usage() { 6 | cat <] 8 | 9 | builds the current distro 10 | 11 | OPTIONS 12 | -l write build-log to given file (instead of ${LOG:-build.log} 13 | -h print this help and exit 14 | 15 | EOF 16 | 17 | } 18 | 19 | while getopts "hl:" opt; do 20 | case "$opt" in 21 | h) 22 | usage 23 | exit 0 24 | ;; 25 | l) 26 | export LOG="$OPTARG" 27 | ;; 28 | ?) 29 | usage 30 | exit 2 31 | ;; 32 | esac 33 | done 34 | shift $(($OPTIND - 1)) 35 | 36 | echo "Distro path: ${DIST_PATH}" 37 | echo "CustomPiOS path: ${CUSTOM_PI_OS_PATH}" 38 | echo "================================================================" 39 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 40 | 41 | 42 | set -x 43 | 44 | ${DIR}/build "$1" 45 | -------------------------------------------------------------------------------- /src/build_docker: -------------------------------------------------------------------------------- 1 | 2 | # Build the docker images that are used to build the CustomPiOS images 3 | 4 | docker build --no-cache=true --force-rm=true --tag=custom_pios -f docker/Dockerfile docker 5 | -------------------------------------------------------------------------------- /src/chroot_in_to_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | source "${DIR}"/common.sh 4 | pushd /distro/workspace 5 | mount_image *.img 2 ./mount boot/firmware 6 | pushd mount 7 | chroot . 8 | popd 9 | popd 10 | -------------------------------------------------------------------------------- /src/config: -------------------------------------------------------------------------------- 1 | CONFIG_DIR=$(dirname $(realpath -s "${BASH_SOURCE}")) 2 | source ${CUSTOM_PI_OS_PATH}/common.sh 3 | 4 | WORKSPACE_POSTFIX= 5 | 6 | export BUILD_VARIANT="" 7 | BUILD_VARIANT="$1" 8 | : ${BUILD_VARIANT:=default} 9 | 10 | EXTRA_BAORD_CONFIG=$2 11 | 12 | export BUILD_FLAVOR="" 13 | # Disable flavor system 14 | #BUILD_FLAVOR="$1" 15 | : ${BUILD_FLAVOR:=default} 16 | 17 | echo -e "--> Building VARIANT $BUILD_VARIANT, FLAVOR $BUILD_FLAVOR" 18 | 19 | # Import the local config if we have one 20 | 21 | if [ -f "${CONFIG_DIR}/config.local" ] 22 | then 23 | echo "Sourcing config.local..." 24 | source "${CONFIG_DIR}/config.local" 25 | fi 26 | 27 | source ${DIST_PATH}/config 28 | 29 | if [ "${BUILD_VARIANT}" != 'default' ]; then 30 | WORKSPACE_POSTFIX="-${BUILD_VARIANT}" 31 | 32 | if [ -d "${DIST_PATH}/variants/${BUILD_VARIANT}" ]; then 33 | export VARIANT_BASE="${DIST_PATH}/variants/${BUILD_VARIANT}" 34 | elif [ -d "${CUSTOM_PI_OS_PATH}/variants/${BUILD_VARIANT}" ]; then 35 | export VARIANT_BASE="${CUSTOM_PI_OS_PATH}/variants/${BUILD_VARIANT}" 36 | else 37 | die "Could not find Variant ${BUILD_VARIANT}" 38 | fi 39 | 40 | if [ "${BUILD_FLAVOR}" = '' ] || [ "${BUILD_FLAVOR}" = 'default' ] 41 | then 42 | VARIANT_CONFIG=${VARIANT_BASE}/config 43 | FLAVOR_CONFIG= 44 | else 45 | VARIANT_CONFIG=${VARIANT_BASE}/config 46 | FLAVOR_CONFIG=${VARIANT_BASE}/config.${BUILD_FLAVOR} 47 | fi 48 | 49 | if [ -n "${FLAVOR_CONFIG}" ] && [ ! -f "${FLAVOR_CONFIG}" ] 50 | then 51 | die "Could not find config file ${FLAVOR_CONFIG}" 52 | fi 53 | fi 54 | 55 | echo Import the variant config if we have one 56 | if [ -n "${VARIANT_CONFIG}" ] && [ -f "${VARIANT_CONFIG}" ] 57 | then 58 | echo "Sourcing variant config ${VARIANT_CONFIG}..." 59 | set -a 60 | source "${VARIANT_CONFIG}" 61 | set +a 62 | fi 63 | 64 | # Import the flavor config if we have one 65 | 66 | if [ -n "${FLAVOR_CONFIG}" ] && [ -f "${FLAVOR_CONFIG}" ] 67 | then 68 | echo "Sourcing flavor config ${FLAVOR_CONFIG}..." 69 | source "${FLAVOR_CONFIG}" 70 | fi 71 | 72 | 73 | 74 | if [ -f "${DIST_PATH}/config.local" ] 75 | then 76 | echo "Sourcing distro config.local..." 77 | source "${DIST_PATH}/config.local" 78 | fi 79 | 80 | # Get only a list 81 | TMP="${MODULES//(/,}" 82 | TMP="${TMP// /}" 83 | MODULES_LIST="${TMP//)/,}" 84 | 85 | 86 | # Base workspace is special, it has to be sourced before the base module, so remote modules could be calcualted 87 | [ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX 88 | # [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script 89 | [ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount 90 | 91 | # Import remote and submodules config 92 | if [ -f "${EXTRA_BAORD_CONFIG}" ]; then 93 | source "${EXTRA_BAORD_CONFIG}" 94 | else 95 | echo "Note: Not sourceing board config" 96 | fi 97 | 98 | export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config 99 | # Remote modules and meta modulese go in first if they want to change standard behaviour 100 | if [ -f "${REMOTE_AND_META_CONFIG}" ]; then 101 | source "${REMOTE_AND_META_CONFIG}" 102 | fi 103 | 104 | load_module_config "${MODULES_LIST}" 105 | -------------------------------------------------------------------------------- /src/config_sanity: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -n "$MODULES" ]; then 3 | echo "Modules not configured in ${DIST_PATH}/config" 4 | exit -1 5 | fi 6 | 7 | exit 0 8 | -------------------------------------------------------------------------------- /src/custompios: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # OctoPi generation script 3 | # This script takes a Raspbian image and adds to it octoprint and verions addons 4 | # Written by Guy Sheffer 5 | # GPL V3 6 | set -e 7 | 8 | export LC_ALL=C 9 | 10 | source "${CUSTOM_PI_OS_PATH}"/common.sh 11 | 12 | echo_green -e "\nBUILD STARTED @ $(date)!\n" 13 | 14 | function execute_chroot_script() { 15 | 16 | # In docker, these extra commands are required to enable this black-magic 17 | if [ -f /.dockerenv ] && [ "$(uname -m)" != "armv7l" ] && [ "$(uname -m)" != "aarch64" ] ; then 18 | if [ "$BASE_ARCH" == "armv7l" ] || [ "$BASE_ARCH" == "armhf" ]; then 19 | update-binfmts --enable qemu-arm 20 | elif [ "$BASE_ARCH" == "aarch64" ] || [ "$BASE_ARCH" == "arm64" ]; then 21 | update-binfmts --enable qemu-aarch64 22 | fi 23 | if ! mount | grep -q "/proc/sys/fs/binfmt_misc"; then 24 | mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true 25 | fi 26 | fi 27 | #move filesystem files 28 | if [ -d "$1/filesystem" ]; then 29 | cp -vr --preserve=mode,timestamps "$1/filesystem" . 30 | fi 31 | 32 | #black magic of qemu-arm-static 33 | # cp `which qemu-arm-static` usr/bin 34 | if [ "$(uname -m)" != "armv7l" ] || [ "$(uname -m)" != "aarch64" ] ; then 35 | if [ "$BASE_ARCH" == "armv7l" ] || [ "$BASE_ARCH" == "armhf" ]; then 36 | if (grep -q gentoo /etc/os-release);then 37 | ROOT="`realpath .`" emerge --usepkgonly --oneshot --nodeps qemu 38 | else 39 | cp `which qemu-arm-static` usr/bin/qemu-arm-static 40 | fi 41 | elif [ "$BASE_ARCH" == "aarch64" ] || [ "$BASE_ARCH" == "arm64" ]; then 42 | if (grep -q gentoo /etc/os-release);then 43 | ROOT="`realpath .`" emerge --usepkgonly --oneshot --nodeps qemu 44 | else 45 | cp `which qemu-aarch64-static` usr/bin/qemu-aarch64-static 46 | fi 47 | fi 48 | fi 49 | 50 | cp $2 chroot_script 51 | chmod 755 chroot_script 52 | cp "${CUSTOM_PI_OS_PATH}"/common.sh common.sh 53 | chmod 755 common.sh 54 | chroot_correct_qemu "$(uname -m)" "$BASE_ARCH" "$2" "${CUSTOM_PI_OS_PATH}" 55 | 56 | # Handle exported items 57 | if [ -d "custompios_export" ]; then 58 | echo "Exporting files from chroot" 59 | echo "List of archies to create:" 60 | ls custompios_export 61 | # Tar files listed in export 62 | for export_list in custompios_export/* ; do 63 | tar --absolute-names -czvf "${BASE_WORKSPACE}/$(basename ${export_list}).tar.gz" -T ${export_list} 64 | done 65 | 66 | rm -rf custompios_export 67 | fi 68 | 69 | #cleanup 70 | rm chroot_script 71 | if [ -d "filesystem" ]; then 72 | rm -rfv "filesystem" 73 | fi 74 | } 75 | 76 | # check prerequisites 77 | if [ -n "$BASE_IMAGE_ENLARGEROOT" ] || [ -n "$BASE_IMAGE_RESIZEROOT" ]; then 78 | # resizing the root partition requires 'sfdisk' in our path 79 | which sfdisk >/dev/null 2>/dev/null || \ 80 | die "'sfdisk' not found in PATH; did you mean to run the script as root?" 81 | fi 82 | 83 | 84 | # start! 85 | 86 | 87 | mkdir -p $BASE_WORKSPACE 88 | mkdir -p $BASE_MOUNT_PATH 89 | 90 | # This is already genrated at "build" sourced in "config", but copying here mostly for debug 91 | if [ -f "${EXTRA_BAORD_CONFIG}" ]; then 92 | mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config 93 | fi 94 | 95 | # Clean exported artifacts from other builds 96 | rm -rf "${BASE_WORKSPACE}"/*.tar.gz 97 | 98 | install_cleanup_trap 99 | install_fail_on_error_trap 100 | unmount_image $BASE_MOUNT_PATH force || true 101 | 102 | pushd "${BASE_WORKSPACE}" 103 | if [ -e *.img ]; then 104 | rm *.img 105 | fi 106 | if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then 107 | echo "Error: could not find image: $BASE_ZIP_IMG" 108 | echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically" 109 | exit 1 110 | fi 111 | 112 | if [[ $BASE_ZIP_IMG =~ \.img$ ]]; then 113 | # if the image is already extracted copy over 114 | cp "$BASE_ZIP_IMG" . 115 | else 116 | 7za x -aoa "$BASE_ZIP_IMG" 117 | fi 118 | 119 | BASE_IMG_PATH=`ls | grep '.img$\|.raw$' | head -n 1` 120 | if [ ! -f "$BASE_IMG_PATH" ]; then 121 | echo "Error, can't find image path, did you place an image in the image folder?" 122 | exit 1 123 | fi 124 | export CUSTOM_PI_OS_BUILDBASE=$(basename "$BASE_IMG_PATH") 125 | 126 | if [ -n "$BASE_IMAGE_ENLARGEROOT" ] 127 | then 128 | # make our image a bit larger so we don't run into size problems... 129 | enlarge_ext $BASE_IMG_PATH $BASE_ROOT_PARTITION $BASE_IMAGE_ENLARGEROOT 130 | fi 131 | 132 | # mount root and boot partition 133 | mount_image "${BASE_IMG_PATH}" "${BASE_ROOT_PARTITION}" "${BASE_MOUNT_PATH}" "${BASE_BOOT_MOUNT_PATH}" "${BASE_BOOT_PARTITION}" 134 | if [ -n "$BASE_APT_CACHE" ] && [ "$BASE_APT_CACHE" != "no" ] 135 | then 136 | mkdir -p "$BASE_APT_CACHE" 137 | mount --bind "$BASE_APT_CACHE" $BASE_MOUNT_PATH/var/cache/apt 138 | fi 139 | 140 | #Edit pi filesystem 141 | pushd $BASE_MOUNT_PATH 142 | 143 | #make QEMU boot (remember to return) 144 | if [ "$BASE_IMAGE_RASPBIAN" == "yes" ]; then 145 | fixLd 146 | fi 147 | #sed -i 's@include /etc/ld.so.conf.d/\*.conf@\#include /etc/ld.so.conf.d/\*.conf@' etc/ld.so.conf 148 | 149 | 150 | ### Execute chroot scripts ### 151 | 152 | # if an additional pre-script is defined, execute that now 153 | if [ -n "$BASE_PRESCRIPT" ] && [ -f $BASE_PRESCRIPT/chroot_script ]; then 154 | echo "Injecting environment pre script from $BASE_PRESCRIPT..." 155 | execute_chroot_script $BASE_PRESCRIPT $BASE_PRESCRIPT/chroot_script 156 | fi 157 | 158 | # if building a variant, execute its pre-chroot script 159 | if [ -n "$VARIANT_BASE" ] && [ -f $VARIANT_BASE/pre_chroot_script ]; then 160 | echo "Injecting variant pre script from $VARIANT_BASE..." 161 | execute_chroot_script $VARIANT_BASE $VARIANT_BASE/pre_chroot_script 162 | fi 163 | 164 | # execute the base chroot script 165 | ### execute_chroot_script $BASE_SCRIPT_PATH $BASE_CHROOT_SCRIPT_PATH 166 | CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script 167 | MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after 168 | MODULES_BEFORE="${MODULES}" 169 | ${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}" 170 | if [ -f "${REMOTE_AND_META_CONFIG}" ]; then 171 | echo "Sourcing remote and submodules config" 172 | source "${REMOTE_AND_META_CONFIG}" ${@} 173 | 174 | MODULES_AFTER=$(cat "${MODULES_AFTER_PATH}") 175 | load_module_config "${MODULES_AFTER}" 176 | 177 | else 178 | echo "No remote and submodules config detected" 179 | fi 180 | echo $ARMBIAN_CONFIG_TXT_FILE 181 | # if you need anything from common running in execute_chroot_script, export it here 182 | export -f chroot_correct_qemu 183 | export -f execute_chroot_script 184 | bash -x "${CHROOT_SCRIPT}" 185 | 186 | 187 | # if building a variant, execute its post-chroot script 188 | if [ -n "$VARIANT_BASE" ] && [ -f $VARIANT_BASE/post_chroot_script ]; then 189 | echo "Injecting variant post script from $VARIANT_BASE..." 190 | execute_chroot_script $VARIANT_BASE $VARIANT_BASE/post_chroot_script 191 | fi 192 | 193 | # if an additional post-script is defined, execute that now 194 | if [ -n "$BASE_POSTSCRIPT" ] && [ -f $BASE_POSTSCRIPT/chroot_script ]; then 195 | echo "Injecting environment post script from $BASE_POSTSCRIPT..." 196 | execute_chroot_script $BASE_POSTSCRIPT $BASE_POSTSCRIPT/chroot_script 197 | fi 198 | 199 | ### End Execute chroot scripts ### 200 | if [ "$BASE_IMAGE_RASPBIAN" == "yes" ]; then 201 | restoreLd 202 | fi 203 | popd 204 | 205 | # unmount first boot, then root partition 206 | unmount_image $BASE_MOUNT_PATH 207 | chmod 644 $BASE_IMG_PATH 208 | 209 | if [ -n "$BASE_IMAGE_RESIZEROOT" ] 210 | then 211 | # resize image to minimal size + provided size 212 | minimize_ext $BASE_IMG_PATH $BASE_ROOT_PARTITION $BASE_IMAGE_RESIZEROOT 213 | fi 214 | popd 215 | 216 | echo_green -e "\nBUILD SUCCEEDED @ $(date)!\n" 217 | -------------------------------------------------------------------------------- /src/custompios_core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guysoft/CustomPiOS/0f05cdaa6197ab0437fd286abc94b00e1899ac69/src/custompios_core/__init__.py -------------------------------------------------------------------------------- /src/custompios_core/base_image_downloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import yaml 4 | import os 5 | import urllib.request 6 | import tempfile 7 | import hashlib 8 | import shutil 9 | import re 10 | import urllib.parse 11 | from enum import Enum, auto 12 | from typing import Dict, Any, Optional, cast, Tuple 13 | from common import get_image_config, read_images 14 | PRECENT_PROGRESS_SIZE = 5 15 | 16 | class ChecksumType(Enum): 17 | URL = auto() # a url for the checksum file 18 | STRING = auto() # A string in the format "checksum filename" 19 | 20 | class ChecksumFailException(Exception): 21 | pass 22 | 23 | RETRY = 3 24 | 25 | def ensure_dir(d, chmod=0o777): 26 | """ 27 | Ensures a folder exists. 28 | Returns True if the folder already exists 29 | """ 30 | if not os.path.exists(d): 31 | os.makedirs(d, chmod) 32 | os.chmod(d, chmod) 33 | return False 34 | return True 35 | 36 | 37 | def download_webpage(url: str) -> Optional[str]: 38 | try: 39 | with urllib.request.urlopen(url) as response: 40 | # Decode the response to a string 41 | webpage = response.read().decode('utf-8') 42 | return webpage 43 | except Exception as e: 44 | print(str(e)) 45 | return None 46 | 47 | def get_location_header(url: str) -> str: 48 | try: 49 | with urllib.request.urlopen(url) as response: 50 | response_url = response.url 51 | 52 | if response_url is None: 53 | raise Exception("Location header is None, can't determine latest rpi image") 54 | return response_url 55 | except Exception as e: 56 | print(str(e)) 57 | print("Error: Failed to determine latest rpi image") 58 | raise e 59 | 60 | 61 | class DownloadProgress: 62 | last_precent: float = 0 63 | def show_progress(self, block_num, block_size, total_size): 64 | new_precent = round(block_num * block_size / total_size * 100, 1) 65 | if new_precent > self.last_precent + PRECENT_PROGRESS_SIZE: 66 | print(f"{new_precent}%", end="\r") 67 | self.last_precent = new_precent 68 | 69 | def get_file_name(headers, url): 70 | if "Content-Disposition" in headers.keys(): 71 | return re.findall("filename=(\S+)", headers["Content-Disposition"])[0] 72 | return url.split('/')[-1] 73 | 74 | def get_sha256(filename): 75 | sha256_hash = hashlib.sha256() 76 | with open(filename,"rb") as f: 77 | for byte_block in iter(lambda: f.read(4096),b""): 78 | sha256_hash.update(byte_block) 79 | file_checksum = sha256_hash.hexdigest() 80 | return file_checksum 81 | return 82 | 83 | def download_image_http(board: Dict[str, Any], dest_folder: str): 84 | url = board["url"] 85 | checksum = board["checksum"] 86 | download_http(url, checksum, dest_folder) 87 | 88 | def download_http(url: str, checksum_argument: str, dest_folder: str, checksum_type: ChecksumType = ChecksumType.URL): 89 | with tempfile.TemporaryDirectory() as tmpdirname: 90 | print('created temporary directory', tmpdirname) 91 | temp_file_name = os.path.join(tmpdirname, "image.xz") 92 | temp_file_checksum = os.path.join(tmpdirname, "checksum.sha256") 93 | 94 | for r in range(RETRY): 95 | try: 96 | # Get sha and confirm its the right image 97 | download_progress = DownloadProgress() 98 | 99 | # We need to get the checksum as one of ChecksumType enum, the result goes in to online_checksum 100 | online_checksum = None 101 | if checksum_type == ChecksumType.URL: 102 | _, headers_checksum = urllib.request.urlretrieve(checksum_argument, temp_file_checksum, download_progress.show_progress) 103 | file_name_checksum = get_file_name(headers_checksum, checksum_argument) 104 | 105 | checksum_data = None 106 | with open(temp_file_checksum, 'r') as f: 107 | checksum_data = f.read() 108 | 109 | checksum_data_parsed = [x.strip() for x in checksum_data.split()] 110 | 111 | elif checksum_type == ChecksumType.STRING: 112 | checksum_data_parsed = checksum_argument.split(" ") 113 | else: 114 | print("Error: provided a non-existant checksum type") 115 | exit(1) 116 | online_checksum = checksum_data_parsed[0] 117 | file_name_from_checksum = checksum_data_parsed[1] 118 | dest_file_name = os.path.join(dest_folder, file_name_from_checksum) 119 | print(f"Downloading {dest_file_name} from {url}") 120 | 121 | if os.path.isfile(dest_file_name): 122 | file_checksum = get_sha256(dest_file_name) 123 | if file_checksum == online_checksum: 124 | print("We got base image file and checksum is right") 125 | return 126 | # Get the file 127 | download_progress = DownloadProgress() 128 | _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress) 129 | 130 | file_name = get_file_name(headers, url) 131 | file_checksum = get_sha256(temp_file_name) 132 | if file_checksum != online_checksum: 133 | print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}') 134 | continue 135 | ensure_dir(os.path.dirname(dest_file_name)) 136 | shutil.move(temp_file_name, dest_file_name) 137 | 138 | except Exception as e: 139 | if r < 2: 140 | print(f'Failed. Attempt # {r + 1}, got: {e}') 141 | else: 142 | print('Error encoutered at {RETRY} attempt') 143 | print(e) 144 | exit(1) 145 | else: 146 | print(f"Success: {temp_file_name}") 147 | break 148 | return 149 | 150 | 151 | def download_image_rpi(board: Dict[str, Any], dest_folder: str): 152 | port = board.get("port", "lite_armhf") 153 | os_name = f"raspios" 154 | distribution = board.get("distribution", "bookworm") 155 | version_file = board.get("version_file", "latest") 156 | version_folder = board.get("version_folder", "latest") 157 | 158 | latest_url = f"https://downloads.raspberrypi.org/{os_name}_{port}_latest" 159 | 160 | download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz" 161 | if version_file == "latest" or version_folder == "latest": 162 | download_url = get_location_header(latest_url) 163 | 164 | checksum_url = f"{download_url}.sha256" 165 | download_http(download_url, checksum_url, dest_folder) 166 | return 167 | 168 | def get_checksum_libre_computer(os_name: str, os_version: str, file_name: str) -> Optional[str]: 169 | checksum_url = f"https://distro.libre.computer/ci/{os_name}/{os_version}/SHA256SUMS" 170 | checksum_files_data = download_webpage(checksum_url) 171 | for line in checksum_files_data.splitlines(): 172 | checksum, name = line.split(maxsplit=1) 173 | print(name) 174 | if name == file_name: 175 | return f"{checksum} {file_name}" 176 | return None 177 | 178 | 179 | def download_image_libre_computer(board: Dict[str, Any], dest_folder: str): 180 | # URL example: https://distro.libre.computer/ci/raspbian/12/2023-10-10-raspbian-bookworm-arm64%2Baml-s905x-cc.img.xz 181 | # URL example: https://distro.libre.computer/ci/debian/12/debian-12-base-arm64%2Baml-s905x-cc.img.xz 182 | port = board.get("port", "base") 183 | arch = board.get("arch", "arm64+aml") 184 | distribution = board.get("distribution", "bookworm") 185 | os_name = board.get("os_name", "debian") 186 | os_version = board.get("os_version", "12") 187 | board = "s905x-cc" 188 | 189 | # download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz" 190 | file_name = None 191 | if os_name == "debian": 192 | file_name = f"{os_name}-{os_version}-{port}-{arch}-{board}.img.xz" 193 | download_url = f"https://distro.libre.computer/ci/{os_name}/{os_version}/{urllib.parse.quote(file_name)}" 194 | elif os_name == "raspbian": 195 | download_url = f"https://distro.libre.computer/ci/{os_name}/{os_version}/{urllib.parse.quote(file_name)}" 196 | checksum = get_checksum_libre_computer(os_name, os_version, file_name) 197 | if checksum is None: 198 | print(f"Error: Can't find the correct checksum for {file_name}") 199 | exit(1) 200 | 201 | download_http(download_url, checksum, dest_folder, ChecksumType.STRING) 202 | return 203 | 204 | if __name__ == "__main__": 205 | parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O') 206 | parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder") 207 | parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256') 208 | args = parser.parse_args() 209 | 210 | images = read_images() 211 | 212 | base_board = os.environ.get("BASE_BOARD", None) 213 | base_image_path = os.environ.get("BASE_IMAGE_PATH", None) 214 | 215 | if base_image_path is None: 216 | print(f'Error: did not find image config file') 217 | exit(1) 218 | cast(str, base_image_path) 219 | 220 | image_config = get_image_config() 221 | if image_config is not None: 222 | if image_config["type"] == "http": 223 | print(f"Downloading image for {base_board}") 224 | download_image_http(image_config, base_image_path) 225 | download_image_http(image_config, base_image_path) 226 | elif image_config["type"] == "rpi": 227 | print(f"Downloading Raspberry Pi image for {base_board}") 228 | download_image_rpi(image_config, base_image_path) 229 | elif image_config["type"] == "libre.computer": 230 | print(f"Downloading image for {base_board}") 231 | download_image_libre_computer(image_config, base_image_path) 232 | elif image_config["type"] == "torrent": 233 | print("Error: Torrent not implemented") 234 | exit(1) 235 | else: 236 | print(f'Error: Unsupported image download type: {image_config["type"]}') 237 | exit(1) 238 | else: 239 | print(f"Error: Image config not found for: {base_board}") 240 | exit(1) 241 | 242 | 243 | print("Done") 244 | -------------------------------------------------------------------------------- /src/custompios_core/common.py: -------------------------------------------------------------------------------- 1 | """ Common functions between CustomPiOS python scripts""" 2 | from typing import Dict, Any, Optional, cast 3 | import yaml 4 | import os 5 | from pathlib import Path 6 | 7 | def get_custompios_folder(): 8 | custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None) 9 | if custompios_path is not None: 10 | return Path(custompios_path) 11 | return Path(__file__).parent.parent 12 | 13 | 14 | IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml") 15 | 16 | 17 | def read_images() -> Dict[str, Dict[str,str]]: 18 | if not os.path.isfile(IMAGES_CONFIG): 19 | raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}") 20 | with open(IMAGES_CONFIG,'r') as f: 21 | output = yaml.safe_load(f) 22 | return output 23 | 24 | def get_image_config() -> Optional[Dict["str", Any]]: 25 | images = read_images() 26 | 27 | base_board = os.environ.get("BASE_BOARD", None) 28 | base_image_path = os.environ.get("BASE_IMAGE_PATH", None) 29 | 30 | # Default to raspberrypiarmhf board in case of CustomPiOS v1 31 | if base_board is None: 32 | base_board = "raspberrypiarmhf" 33 | 34 | if base_board is not None and base_board in images["images"]: 35 | return images["images"][base_board] 36 | return None 37 | -------------------------------------------------------------------------------- /src/custompios_core/execution_order.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | #a='base(octopi,a(b,c(a2)),mm)' 3 | import argparse 4 | import os 5 | import subprocess 6 | from get_remote_modules import get_remote_module 7 | from typing import TextIO, List, Tuple, Dict, Any, cast, Union 8 | 9 | 10 | def run_command(command: List[str], **kwargs: Dict[str, Any]): 11 | is_timeout = False 12 | p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) # type: ignore 13 | try: 14 | stdout, stderr = p.communicate(timeout=5) 15 | except subprocess.TimeoutExpired as e: 16 | p.kill() 17 | stdout,stderr = p.communicate() 18 | is_timeout = True 19 | try: 20 | stdout = stdout.decode("utf-8") 21 | except UnicodeDecodeError as e: 22 | print("Error: can't decode stdout") 23 | print(e) 24 | print(stdout) 25 | stdout = "" 26 | 27 | try: 28 | stderr = stderr.decode("utf-8") 29 | except UnicodeDecodeError as e: 30 | print("Error: can't decode stderr") 31 | print(stderr) 32 | print(e) 33 | stderr = "" 34 | 35 | return stdout, stderr, is_timeout 36 | 37 | 38 | def write_modules_scripts(module: str, state: str, module_folder: str, out: TextIO): 39 | out.write("# " + state + "_" + module + "\n") 40 | script = os.path.join(module_folder, state + "_chroot_script") 41 | if os.path.isfile(script): 42 | out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n") 43 | else: 44 | print("WARNING: No file at - " + script) 45 | 46 | return 47 | 48 | def parse(a: str) -> List[Tuple[str,str]]: 49 | stack=[] 50 | return_value = [] 51 | token = "" 52 | 53 | for char in a: 54 | if char == "(": 55 | stack.append(token) 56 | if token != "": 57 | return_value.append((token, "start")) 58 | token = "" 59 | elif char == ")": 60 | parent = stack.pop() 61 | if token != "": 62 | return_value.append((token, "start")) 63 | return_value.append((token, "end")) 64 | token = "" 65 | if parent != "": 66 | return_value.append((parent, "end")) 67 | elif char == ",": 68 | if token != "": 69 | return_value.append((token, "start")) 70 | return_value.append((token, "end")) 71 | token = "" 72 | else: 73 | token += char 74 | 75 | if token != "": 76 | return_value.append((token, "start")) 77 | return_value.append((token, "end")) 78 | if len(stack) > 0: 79 | raise Exception(str(stack)) 80 | return return_value 81 | 82 | def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,str]],Dict[str,str]]: 83 | return_value = [] 84 | modules_to_modules_folder = {} 85 | for module, state in modules: 86 | module_folders = [ 87 | os.path.join(os.environ['DIST_PATH'], "modules", module), 88 | os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module) 89 | ] 90 | # In case this is a meta module, order counts 91 | if state == "start": 92 | return_value.append((module, state)) 93 | found_local = False 94 | found_remote = False 95 | for module_folder in module_folders: 96 | if os.path.isdir(module_folder): 97 | found_local = True 98 | modules_to_modules_folder[module] = module_folder 99 | break 100 | 101 | if not found_local and module: 102 | # TODO: Handle update 103 | found_remote, module_folder_remote = get_remote_module(module) 104 | if module_folder_remote is not None: 105 | module_folder = module_folder_remote 106 | 107 | modules_to_modules_folder[module] = module_folder 108 | 109 | 110 | if not found_local and not found_remote: 111 | print(f"Error: Module {module} does not exist and is not in remote modules list") 112 | exit(1) 113 | 114 | meta_module_path = os.path.join(module_folder, "meta") 115 | if os.path.isfile(meta_module_path): 116 | # Meta module detected 117 | print(f"Running: {meta_module_path}") 118 | print(f"ENV: {os.environ['BASE_BOARD']}") 119 | submodules, meta_module_errors, is_timeout = run_command([meta_module_path]) 120 | submodules = submodules.strip() 121 | print(f"Adding in modules: {submodules}") 122 | if meta_module_errors != "" or is_timeout: 123 | print(meta_module_errors) 124 | print(f"Got error processing meta module at: {meta_module_path}") 125 | exit(1) 126 | if submodules != "": 127 | print(f"Got sub modules: {submodules}") 128 | 129 | for sub_module in submodules.split(","): 130 | sub_module = sub_module.strip() 131 | return_value_sub, modules_to_modules_folder_sub = handle_meta_modules([(sub_module, state)]) 132 | return_value += return_value_sub 133 | modules_to_modules_folder.update(modules_to_modules_folder_sub) 134 | # In case this is a meta module, order counts 135 | if state == "end": 136 | return_value.append((module, state)) 137 | 138 | return return_value, modules_to_modules_folder 139 | 140 | 141 | if __name__ == "__main__": 142 | parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules') 143 | parser.add_argument('modules', type=str, help='A string showing how the modules should be called') 144 | parser.add_argument('output_script', type=str, help='path to output the chroot script master') 145 | parser.add_argument('modules_after_path', nargs='?', default=None, type=str, help='path to output the chroot script master') 146 | parser.add_argument('remote_and_meta_config_path', nargs='?', default=None, type=str, help='path to output the config script of remote modules and submodules') 147 | args = parser.parse_args() 148 | 149 | if os.path.isfile(args.output_script): 150 | os.remove(args.output_script) 151 | 152 | with open(args.output_script, "w+") as f: 153 | f.write("#!/usr/bin/env bash\n") 154 | f.write("set -x\n") 155 | f.write("set -e\n") 156 | initial_execution_order = parse(args.modules.replace(" ", "")) 157 | f.write(f"# Defined execution order: {initial_execution_order}\n") 158 | modules_execution_order, modules_to_modules_folder = handle_meta_modules(initial_execution_order) 159 | f.write(f"# With meta modules order: {modules_execution_order}\n") 160 | 161 | for module, state in modules_execution_order: 162 | module_folder = modules_to_modules_folder[module] 163 | write_modules_scripts(module, state, module_folder, f) 164 | 165 | # List all new modules add them in, then remove existing ones 166 | list_new_modules = [] 167 | for module, state in modules_execution_order: 168 | if module not in list_new_modules: 169 | list_new_modules.append(module) 170 | for module, state in initial_execution_order: 171 | if module in list_new_modules: 172 | list_new_modules.remove(module) 173 | 174 | # TODO2: load configs from yaml 175 | if args.modules_after_path is not None: 176 | with open(args.modules_after_path, "w") as w: 177 | w.write(",".join(list_new_modules)) 178 | 179 | with open(args.remote_and_meta_config_path, "w") as f: 180 | for module in list_new_modules: 181 | module_folder = modules_to_modules_folder[module] 182 | module_config_path = os.path.join(module_folder, "config") 183 | if os.path.isfile(module_config_path): 184 | f.write(f"source {module_config_path}\n") 185 | 186 | os.chmod(args.output_script, 0o755) 187 | 188 | -------------------------------------------------------------------------------- /src/custompios_core/generate_board_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import os 3 | import yaml 4 | from pathlib import Path 5 | from typing import Tuple, Optional, Dict, Any, cast 6 | import git 7 | from git import RemoteProgress 8 | from common import get_image_config 9 | import argparse 10 | import sys 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser(add_help=True, description='Create an export shell script to use the yaml-configured variables') 14 | parser.add_argument('output_script', type=str, help='path to output the chroot script master') 15 | args = parser.parse_args() 16 | image_config = get_image_config() 17 | if image_config is None: 18 | print("Error: Could not get image config") 19 | sys.exit(1) 20 | cast(Dict[str,Any], image_config) 21 | if not "env" in image_config.keys(): 22 | print("Warning: no env in image config") 23 | exit() 24 | env = image_config["env"] 25 | with open(args.output_script, "w+") as w: 26 | for key in env.keys(): 27 | w.write(f'export {key}="{env[key]}"\n') 28 | 29 | -------------------------------------------------------------------------------- /src/custompios_core/get_remote_modules.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yaml 3 | from pathlib import Path 4 | from typing import Tuple, Optional 5 | import git 6 | from git import RemoteProgress 7 | from common import get_custompios_folder 8 | 9 | # TODO add env var to set this 10 | REMOTES_DIR = os.path.join(get_custompios_folder(), "remotes") 11 | REMOTE_CONFIG = os.path.join(get_custompios_folder(), "modules_remote.yml") 12 | 13 | 14 | class CloneProgress(RemoteProgress): 15 | def update(self, op_code, cur_count, max_count=None, message=''): 16 | if message: 17 | print(message) 18 | 19 | 20 | def ensure_dir(d, chmod=0o777): 21 | """ 22 | Ensures a folder exists. 23 | Returns True if the folder already exists 24 | """ 25 | if not os.path.exists(d): 26 | os.makedirs(d, chmod) 27 | os.chmod(d, chmod) 28 | return False 29 | return True 30 | 31 | 32 | def read_remotes(): 33 | if not os.path.isfile(REMOTE_CONFIG): 34 | raise Exception(f"Error: Remotes config file not found: {REMOTE_CONFIG}") 35 | with open(REMOTE_CONFIG,'r') as f: 36 | output = yaml.safe_load(f) 37 | return output 38 | 39 | def get_remote_module(module: str) -> Tuple[bool, Optional[str]]: 40 | """ Gets the remote module and saves it to cache. Returns True if found, else false""" 41 | print(f'INFO: Module "{module}", looking for remote module and downloading') 42 | modules_remotes = read_remotes() 43 | print(modules_remotes.keys()) 44 | 45 | if "modules" not in modules_remotes.keys() and module not in modules_remotes["modules"].keys(): 46 | return False, None 47 | 48 | ensure_dir(REMOTES_DIR) 49 | 50 | if "remotes" not in modules_remotes.keys() or module not in modules_remotes["modules"].keys(): 51 | return False, None 52 | 53 | module_config = modules_remotes["modules"][module] 54 | 55 | remote_for_module = module_config["remote"] 56 | remote_config = modules_remotes["remotes"][remote_for_module] 57 | 58 | if remote_config.get("type", "git") == "git": 59 | if "repo" not in remote_config.keys(): 60 | print(f'Error: repo field not set for remote: "{remote_for_module}" used by remote module "{module}"') 61 | return False, None 62 | 63 | if "tag" not in remote_config.keys(): 64 | print(f'Error: repo tag field not set for remote: "{remote_for_module}" used by remote module "{module}"') 65 | return False, None 66 | 67 | repo_url = remote_config["repo"] 68 | branch = remote_config["tag"] 69 | 70 | # credentials = base64.b64encode(f"{GHE_TOKEN}:".encode("latin-1")).decode("latin-1") 71 | # TODO: Handle update of remote 72 | remote_to_path = os.path.join(REMOTES_DIR, remote_for_module) 73 | if not os.path.exists(remote_to_path): 74 | git.Repo.clone_from( 75 | url=repo_url, 76 | single_branch=True, 77 | depth=1, 78 | to_path=f"{remote_to_path}", 79 | branch=branch, 80 | ) 81 | 82 | if "path" not in module_config.keys(): 83 | print(f"Error: repo tag field not set for remote: {remote_for_module} used by remote module {module}") 84 | return False, None 85 | module_path = os.path.join(remote_to_path, module_config["path"]) 86 | return True, module_path 87 | 88 | else: 89 | print(f"Error: unsupported type {modules_remotes[module]['type']} for module {module}") 90 | return False, None 91 | return False, None 92 | -------------------------------------------------------------------------------- /src/custompios_core/list_boards.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | from common import read_images 3 | 4 | if __name__ == "__main__": 5 | images = read_images()["images"] 6 | print("Available board targest for --board are:") 7 | for key in sorted(images): 8 | if "description" in images[key].keys(): 9 | print(f'{key} - {images[key]["description"]}') 10 | else: 11 | print(key) 12 | -------------------------------------------------------------------------------- /src/custompios_core/make_rpi-imager-snipplet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import json 3 | import zipfile 4 | import hashlib 5 | import os 6 | import argparse 7 | from datetime import date 8 | import glob 9 | from typing import Optional, Dict, Union 10 | 11 | 12 | 13 | def handle_arg(key, optional=False): 14 | if optional and key not in os.environ.keys(): 15 | return 16 | if key in os.environ.keys(): 17 | return os.environ[key] 18 | else: 19 | print("Error: Missing value in your distro config file for rpi-imager json generator: " + str(key)) 20 | exit(1) 21 | 22 | 23 | if __name__ == "__main__": 24 | parser = argparse.ArgumentParser(add_help=True, description='Create a json snipplet from an image to be used with the make_rpi-imager_list.py and eventually published in a repo') 25 | parser.add_argument('workspace_suffix', nargs='?', default="default", type=str, help='Suffix of workspace folder') 26 | parser.add_argument('-u', '--rpi_imager_url', type=str, default="MISSING_URL", help='url to the uploaded image url') 27 | 28 | args = parser.parse_args() 29 | 30 | workspace_path = os.path.join(os.getcwd(), "workspace") 31 | if args.workspace_suffix != "" and args.workspace_suffix != "default": 32 | workspace_path += "-" + args.workspace_suffix 33 | 34 | name = handle_arg("RPI_IMAGER_NAME") 35 | description = handle_arg("RPI_IMAGER_DESCRIPTION") 36 | url = args.rpi_imager_url 37 | icon = handle_arg("RPI_IMAGER_ICON") 38 | website = handle_arg("RPI_IMAGER_WEBSITE", True) 39 | release_date = date.today().strftime("%Y-%m-%d") 40 | zip_local = glob.glob(os.path.join(workspace_path,"*.zip"))[0] 41 | 42 | if url == "MISSING_URL": 43 | url = os.path.basename(zip_local) 44 | 45 | 46 | output_path = os.path.join(workspace_path, "rpi-imager-snipplet.json") 47 | 48 | json_out: Dict[str, Optional[Union[str, int]]] = { 49 | "name": name, 50 | "description": description, 51 | "url": url, 52 | "icon": icon, 53 | "release_date": release_date, 54 | } 55 | 56 | if website is not None: 57 | json_out["website"] = website 58 | 59 | img_sha256_path = glob.glob(os.path.join(workspace_path,"*.img.sha256"))[0] 60 | json_out["extract_sha256"] = None 61 | with open(img_sha256_path, 'r') as f: 62 | json_out["extract_sha256"] = f.read().split()[0] 63 | 64 | json_out["extract_size"] = None 65 | with zipfile.ZipFile(zip_local) as zip_file: 66 | json_out["extract_size"] = zip_file.filelist[0].file_size 67 | 68 | json_out["image_download_size"] = os.stat(zip_local).st_size 69 | 70 | json_out["image_download_sha256"] = None 71 | with open(zip_local,"rb") as fh: 72 | json_out["image_download_sha256"] = hashlib.sha256(fh.read()).hexdigest() 73 | 74 | with open(output_path, "w") as w: 75 | json.dump(json_out, w, indent=2) 76 | 77 | print("Done generating rpi-imager json snipplet") 78 | -------------------------------------------------------------------------------- /src/custompios_core/multi-arch-manifest.yaml: -------------------------------------------------------------------------------- 1 | image: ghcr.io/guysoft/custompios:devel 2 | manifests: 3 | - image: ghcr.io/guysoft/custompios:amd64 4 | platform: 5 | architecture: amd64 6 | os: linux 7 | - image: ghcr.io/guysoft/custompios:arm32v7 8 | platform: 9 | architecture: arm 10 | os: linux 11 | variant: v7 12 | - image: ghcr.io/guysoft/custompios:arm64v8 13 | platform: 14 | architecture: arm64 15 | os: linux 16 | -------------------------------------------------------------------------------- /src/custompios_core/multi_build.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def get_choices(): 4 | return ['rock', 'paper', 'scissors'] 5 | 6 | def main(): 7 | parser = argparse.ArgumentParser(add_help=True, description='Build mulitple images for multiple devices') 8 | parser.add_argument('--list', "-l", choices=get_choices(), type=str, nargs='+') 9 | args = parser.parse_args() 10 | print(args.list) 11 | print("Done") 12 | return 13 | 14 | if __name__ == "__main__": 15 | main() -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/.gitignore: -------------------------------------------------------------------------------- 1 | /workspace 2 | /config.local 3 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/build_dist: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 3 | 4 | export DIST_PATH=${DIR} 5 | export CUSTOM_PI_OS_PATH=$(<${DIR}/custompios_path) 6 | export PATH=$PATH:$CUSTOM_PI_OS_PATH 7 | 8 | ${CUSTOM_PI_OS_PATH}/build_custom_os $@ 9 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/config: -------------------------------------------------------------------------------- 1 | export DIST_NAME=ExampleOS 2 | export DIST_VERSION=0.1.0 3 | 4 | # rpi-imager json generator settings 5 | export RPI_IMAGER_NAME="${DIST_NAME}" 6 | export RPI_IMAGER_DESCRIPTION="A raspberrypi distro built with CustomPiOS" 7 | export RPI_IMAGER_WEBSITE="https://github.com/guysoft/CustomPiOS" 8 | export RPI_IMAGER_ICON="https://raw.githubusercontent.com/guysoft/CustomPiOS/devel/media/rpi-imager-CustomPiOS.png" 9 | 10 | export MODULES="base(network,example)" 11 | 12 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/image/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !README 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/image/README: -------------------------------------------------------------------------------- 1 | Place zipped Rasbian image here. 2 | 3 | If not otherwise specified, the build script will always use the most 4 | recent zip file matching the file name pattern "*-raspbian.zip" located 5 | here. 6 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/modules/example/config: -------------------------------------------------------------------------------- 1 | EXAMPLE_VAR="This is a module variable" 2 | -------------------------------------------------------------------------------- /src/dist_generators/dist_example/src/modules/example/end_chroot_script: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #