├── tests ├── data │ ├── This is a title.md │ ├── Fotoksiążka-Wzór.xmcf │ │ └── jtymejczyk │ │ │ └── Documents │ │ │ └── Fotografia │ │ │ ├── Fotoksiążka Wzór.xmcf │ │ │ ├── Icon │ │ │ ├── ._Icon │ │ │ ├── ._data.mcf │ │ │ ├── ._data.mcf~ │ │ │ └── thumbnails │ │ │ │ ├── 001.png │ │ │ │ └── ._001.png │ │ │ └── ._Fotoksiążka Wzór.xmcf │ ├── medium.jpeg │ ├── ms.band.zip │ ├── thumb.jpeg │ ├── original.jpeg │ ├── Project.band.zip │ ├── Fotoksiążka-Wzór.xmcf.zip │ ├── ms.band │ │ ├── Alternatives │ │ │ └── 000 │ │ │ │ ├── ProjectData │ │ │ │ ├── MetaData.plist │ │ │ │ ├── WindowImage.jpg │ │ │ │ ├── DisplayState.plist │ │ │ │ └── DisplayStateArchive │ │ └── Resources │ │ │ └── ProjectInformation.plist │ ├── Project_original.band │ │ ├── Alternatives │ │ │ └── 000 │ │ │ │ ├── ProjectData │ │ │ │ ├── MetaData.plist │ │ │ │ ├── WindowImage.jpg │ │ │ │ ├── DisplayState.plist │ │ │ │ └── DisplayStateArchive │ │ └── Resources │ │ │ └── ProjectInformation.plist │ ├── test_config_env.yaml │ └── test_config.yaml ├── .env ├── bandit.yaml ├── test_docker_entrypoint.py ├── __init__.py ├── test_container_integration.py ├── test_src_init.py └── test_sync_stats.py ├── gpg-init.sh ├── Coveragerc ├── init.sh ├── requirements.txt ├── .gitmodules ├── src ├── main.py ├── drive_thread_config.py ├── photo_filter_utils.py ├── filesystem_utils.py ├── photo_cleanup_utils.py ├── drive_cleanup.py ├── drive_folder_processing.py ├── config_logging.py ├── hardlink_registry.py ├── drive_file_download.py ├── drive_package_processing.py ├── config_utils.py ├── photo_file_utils.py ├── sync_drive.py ├── drive_filtering.py ├── drive_file_existence.py ├── email_message.py ├── sync_stats.py ├── photo_path_utils.py ├── drive_parallel_download.py ├── album_sync_orchestrator.py ├── drive_sync_directory.py ├── __init__.py ├── photo_download_manager.py └── usage.py ├── requirements-test.txt ├── pytest.ini ├── .dockerignore ├── .vscode ├── settings.json └── launch.json ├── .github ├── workflows │ ├── close-stale.yml │ ├── official-release-to-docker-hub.yml │ ├── ci-pr-test.yml │ └── ci-main-test-coverage-deploy.yml ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── FUNDING.yml └── copilot-instructions.md ├── run-ci.sh ├── run-in-env.sh ├── Dockerfile-debug ├── .pre-commit-config.yaml ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── generate_badges.py ├── docker-entrypoint.sh ├── .yamllint ├── LICENSE ├── USAGE.md ├── Dockerfile ├── .gitignore ├── unraid └── icloud.xml ├── .ruff.toml ├── config.yaml ├── CODE_OF_CONDUCT.md └── NOTIFICATION_CONFIG.md /tests/data/This is a title.md: -------------------------------------------------------------------------------- 1 | # hello world 2 | -------------------------------------------------------------------------------- /gpg-init.sh: -------------------------------------------------------------------------------- 1 | export GPG_TTY=$(tty) 2 | echo "" | gpg --clearsign -------------------------------------------------------------------------------- /tests/.env: -------------------------------------------------------------------------------- 1 | ENV_CONFIG_FILE_PATH=./tests/data/test_config.yaml -------------------------------------------------------------------------------- /Coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = 3 | src/* 4 | [report] 5 | include = 6 | src/* -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/Icon : -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data/medium.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/medium.jpeg -------------------------------------------------------------------------------- /tests/data/ms.band.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band.zip -------------------------------------------------------------------------------- /tests/data/thumb.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/thumb.jpeg -------------------------------------------------------------------------------- /init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd /app && export PYTHONPATH=/app && export HOME=/home/abc && python ./src/main.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | icloudpy==0.8.0 2 | ruamel.yaml==0.18.16 3 | python-magic==0.4.27 4 | requests~=2.32.3 5 | -------------------------------------------------------------------------------- /tests/data/original.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/original.jpeg -------------------------------------------------------------------------------- /tests/data/Project.band.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project.band.zip -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/icloudpy"] 2 | path = external/icloudpy 3 | url = https://github.com/mandarons/icloudpy.git 4 | -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf.zip -------------------------------------------------------------------------------- /tests/data/ms.band/Alternatives/000/ProjectData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Alternatives/000/ProjectData -------------------------------------------------------------------------------- /tests/data/ms.band/Alternatives/000/MetaData.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Alternatives/000/MetaData.plist -------------------------------------------------------------------------------- /tests/data/ms.band/Alternatives/000/WindowImage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Alternatives/000/WindowImage.jpg -------------------------------------------------------------------------------- /tests/data/ms.band/Alternatives/000/DisplayState.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Alternatives/000/DisplayState.plist -------------------------------------------------------------------------------- /tests/data/ms.band/Resources/ProjectInformation.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Resources/ProjectInformation.plist -------------------------------------------------------------------------------- /tests/data/ms.band/Alternatives/000/DisplayStateArchive: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/ms.band/Alternatives/000/DisplayStateArchive -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | """Main module.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | from src import sync 6 | 7 | if __name__ == "__main__": 8 | sync.sync() 9 | -------------------------------------------------------------------------------- /tests/data/Project_original.band/Alternatives/000/ProjectData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Alternatives/000/ProjectData -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | allure-pytest==2.15.2 3 | pytest==9.0.2 4 | coverage==7.13.0 5 | pytest-cov==7.0.0 6 | ruff==0.14.9 7 | ipython==8.37.0 8 | pre-commit==4.5.0 -------------------------------------------------------------------------------- /tests/data/Project_original.band/Alternatives/000/MetaData.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Alternatives/000/MetaData.plist -------------------------------------------------------------------------------- /tests/data/Project_original.band/Alternatives/000/WindowImage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Alternatives/000/WindowImage.jpg -------------------------------------------------------------------------------- /tests/data/Project_original.band/Resources/ProjectInformation.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Resources/ProjectInformation.plist -------------------------------------------------------------------------------- /tests/data/Project_original.band/Alternatives/000/DisplayState.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Alternatives/000/DisplayState.plist -------------------------------------------------------------------------------- /tests/data/Project_original.band/Alternatives/000/DisplayStateArchive: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Project_original.band/Alternatives/000/DisplayStateArchive -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | minversion = 6.0 3 | testpaths = 4 | tests 5 | addopts = -ra -q --cov --cov-report html --cov-report xml --cov-config=Coveragerc --alluredir=./allure-results --cov-fail-under=100 -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/._Fotoksiążka Wzór.xmcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/._Fotoksiążka Wzór.xmcf -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._Icon : -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._Icon -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._data.mcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._data.mcf -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._data.mcf~: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/._data.mcf~ -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/thumbnails/001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/thumbnails/001.png -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | src/drive/* 2 | venv 3 | htmlcov 4 | .pytest_cache 5 | allure-results 6 | tests 7 | pylintrc 8 | .gitignore 9 | .git 10 | Coveragerc 11 | Dockerfile 12 | allure-report 13 | pytest.ini 14 | run-tests.sh 15 | requirements-test.txt 16 | session 17 | session_data -------------------------------------------------------------------------------- /tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/thumbnails/._001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mandarons/icloud-docker/HEAD/tests/data/Fotoksiążka-Wzór.xmcf/jtymejczyk/Documents/Fotografia/Fotoksiążka Wzór.xmcf/thumbnails/._001.png -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.defaultInterpreterPath": "${workspaceFolder}/.venv", 3 | "python.testing.pytestArgs": ["tests", "--no-cov"], 4 | "python.testing.unittestEnabled": false, 5 | "python.testing.pytestEnabled": true, 6 | "python.envFile": "${workspaceFolder}/tests/.env" 7 | } 8 | -------------------------------------------------------------------------------- /tests/bandit.yaml: -------------------------------------------------------------------------------- 1 | # https://bandit.readthedocs.io/en/latest/config.html 2 | 3 | tests: 4 | - B103 5 | - B108 6 | - B306 7 | - B307 8 | - B313 9 | - B314 10 | - B315 11 | - B316 12 | - B317 13 | - B318 14 | - B319 15 | - B320 16 | - B601 17 | - B602 18 | - B604 19 | - B608 20 | - B609 21 | -------------------------------------------------------------------------------- /.github/workflows/close-stale.yml: -------------------------------------------------------------------------------- 1 | name: "Close stale issues" 2 | on: 3 | schedule: 4 | - cron: "30 1 * * *" 5 | 6 | jobs: 7 | stale: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/stale@v9 11 | with: 12 | stale-issue-message: "This issue is stale because it has been open 1 year with no activity. Remove stale label or comment or this will be closed in 5 days." 13 | days-before-stale: 365 14 | days-before-close: 5 15 | -------------------------------------------------------------------------------- /run-ci.sh: -------------------------------------------------------------------------------- 1 | if [ -d .pytest_cache ]; then rm -rf .pytest_cache; fi 2 | if [ -d htmlcov ]; then rm -rf htmlcov; fi 3 | if [ -d icloud ]; then rm -rf icloud; fi 4 | if [ -d session_data ]; then rm -rf session_data; fi 5 | if [ -f icloud.log ]; then rm -f icloud.log; fi 6 | echo "Ruffing ..." && 7 | ruff check --fix && 8 | echo "Testing ..." && 9 | ENV_CONFIG_FILE_PATH=./tests/data/test_config.yaml pytest && 10 | echo "Reporting ..." && 11 | allure generate --clean && 12 | echo "Done." 13 | -------------------------------------------------------------------------------- /run-in-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -eu 3 | 4 | # Activate pyenv and virtualenv if present, then run the specified command 5 | 6 | # pyenv, pyenv-virtualenv 7 | if [ -s .python-version ]; then 8 | PYENV_VERSION=$(head -n 1 .python-version) 9 | export PYENV_VERSION 10 | fi 11 | 12 | # other common virtualenvs 13 | my_path=$(git rev-parse --show-toplevel) 14 | 15 | for venv in venv .venv .; do 16 | if [ -f "${my_path}/${venv}/bin/activate" ]; then 17 | . "${my_path}/${venv}/bin/activate" 18 | break 19 | fi 20 | done 21 | 22 | exec "$@" -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE]" 5 | labels: enhancement 6 | assignees: mandarons 7 | --- 8 | 9 | **Use case** 10 | As a , I want to so that I can . 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or screenshots about the feature request here. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: mandarons 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | Steps to reproduce the behavior: 14 | 15 | 1. Do this '...' 16 | 2. See error 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Screenshots** 22 | If applicable, add screenshots to help explain your problem. 23 | 24 | **Configuration** 25 | If applicable, please share the configuration details 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: mandarons 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: ["https://www.buymeacoffee.com/mandarons"] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /Dockerfile-debug: -------------------------------------------------------------------------------- 1 | FROM python:3.10-alpine3.19 AS build 2 | RUN apk update && apk add git gcc musl-dev py3-pip python3 python3-dev libffi-dev openssl-dev 3 | COPY requirements.txt . 4 | RUN python3 -m venv /venv 5 | ENV PATH="/venv/bin/:$PATH" 6 | RUN pip3 install -U pip 7 | RUN pip3 install -r requirements.txt 8 | RUN pip3 install debugpy 9 | FROM python:3.10-alpine3.19 10 | COPY --from=build /venv /venv 11 | # Libmagic is required at runtime by python-magic 12 | RUN apk update && apk add libmagic shadow dumb-init 13 | ENV PATH="/venv/bin/:$PATH" 14 | ENV PYTHONPATH /app 15 | # Map local folder to /app instead 16 | #COPY . /app/ 17 | 18 | WORKDIR /app 19 | EXPOSE 5678 20 | ENTRYPOINT ['dumb-init', '--'] 21 | # Run below command 22 | #CMD ["python3", "-m", "debugpy","--listen", "0.0.0.0:5678", "--wait-for-client", "main.py"] 23 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: main.py", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "./src/main.py", 12 | "console": "integratedTerminal", 13 | "justMyCode": false, 14 | "cwd": "${workspaceFolder}", 15 | "env": { 16 | "PYTHONPATH": "${workspaceFolder}" 17 | } 18 | }, 19 | { 20 | "name": "Python: Debug Tests", 21 | "type": "python", 22 | "request": "attach", 23 | "purpose": ["debug-test"], 24 | "justMyCode": false 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /src/drive_thread_config.py: -------------------------------------------------------------------------------- 1 | """Thread configuration utilities. 2 | 3 | This module provides thread configuration functionality for parallel operations, 4 | separating thread management from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | from typing import Any 10 | 11 | from src import config_parser, configure_icloudpy_logging, get_logger 12 | 13 | # Configure icloudpy logging immediately after import 14 | configure_icloudpy_logging() 15 | 16 | LOGGER = get_logger() 17 | 18 | 19 | def get_max_threads(config: Any) -> int: 20 | """Get maximum number of threads for parallel downloads. 21 | 22 | Args: 23 | config: Configuration dictionary 24 | 25 | Returns: 26 | Maximum number of threads to use 27 | """ 28 | return config_parser.get_app_max_threads(config) 29 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | rev: v0.13.3 4 | hooks: 5 | - id: ruff 6 | args: 7 | - --fix 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v6.0.0 10 | hooks: 11 | - id: check-executables-have-shebangs 12 | stages: [manual] 13 | - id: check-json 14 | exclude: (.vscode|.devcontainer) 15 | - id: no-commit-to-branch 16 | args: 17 | - --branch=main 18 | - repo: https://github.com/adrienverge/yamllint.git 19 | rev: v1.37.1 20 | hooks: 21 | - id: yamllint 22 | - repo: https://github.com/PyCQA/bandit 23 | rev: 1.8.6 24 | hooks: 25 | - id: bandit 26 | args: 27 | - --quiet 28 | - --format=custom 29 | - --configfile=tests/bandit.yaml 30 | files: ^(src|tests)/.+\.py$ 31 | -------------------------------------------------------------------------------- /src/photo_filter_utils.py: -------------------------------------------------------------------------------- 1 | """Photo filtering utilities module. 2 | 3 | This module contains utilities for photo filtering and validation 4 | during photo synchronization. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | 10 | from src import get_logger 11 | 12 | LOGGER = get_logger() 13 | 14 | 15 | def is_photo_wanted(photo, extensions: list[str] | None) -> bool: 16 | """Check if photo is wanted based on extension filters. 17 | 18 | Args: 19 | photo: Photo object from iCloudPy 20 | extensions: List of allowed file extensions, None means all extensions allowed 21 | 22 | Returns: 23 | True if photo should be synced, False otherwise 24 | """ 25 | if not extensions or len(extensions) == 0: 26 | return True 27 | 28 | for extension in extensions: 29 | if photo.filename.lower().endswith(str(extension).lower()): 30 | return True 31 | return False 32 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.10 2 | 3 | SHELL ["/bin/bash", "-o", "pipefail", "-c"] 4 | 5 | RUN \ 6 | apt-get update \ 7 | && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 8 | software-properties-common git default-jre && \ 9 | apt-get clean &&\ 10 | rm -rf /var/lib/apt/lists/* 11 | # Install allure report 12 | RUN \ 13 | wget https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/2.20.1/allure-commandline-2.20.1.zip && \ 14 | unzip allure-commandline-2.20.1.zip -d /allure && \ 15 | rm allure-commandline-2.20.1.zip 16 | 17 | RUN mkdir /config /icloud && chown -R vscode:vscode /config /icloud 18 | USER vscode 19 | # Install uv (pip replacement) 20 | RUN \ 21 | curl -LsSf https://astral.sh/uv/install.sh | sh 22 | 23 | ENV PATH="/allure/allure-2.20.1/bin:/home/vscode/.cargo/bin:${PATH}" 24 | 25 | WORKDIR /workspaces 26 | -------------------------------------------------------------------------------- /src/filesystem_utils.py: -------------------------------------------------------------------------------- 1 | """File system utilities for directory operations. 2 | 3 | Provides reusable functions for directory creation and path manipulation, 4 | separated from configuration logic per Single Responsibility Principle. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | 11 | 12 | def ensure_directory_exists(path: str) -> str: 13 | """Create directory if it doesn't exist and return absolute path. 14 | 15 | Args: 16 | path: Directory path to create 17 | 18 | Returns: 19 | Absolute path to the created/existing directory 20 | """ 21 | abs_path = os.path.abspath(path) 22 | os.makedirs(abs_path, exist_ok=True) 23 | return abs_path 24 | 25 | 26 | def join_and_ensure_path(base_path: str, *paths: str) -> str: 27 | """Join paths and ensure the resulting directory exists. 28 | 29 | Args: 30 | base_path: Base directory path 31 | *paths: Additional path components to join 32 | 33 | Returns: 34 | Absolute path to the created/existing directory 35 | """ 36 | full_path = os.path.join(base_path, *paths) 37 | return ensure_directory_exists(full_path) 38 | -------------------------------------------------------------------------------- /generate_badges.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import shutil 4 | import xml.etree.ElementTree as ET 5 | 6 | import requests 7 | 8 | badges_directory = "./badges" 9 | 10 | with open("./allure-report/widgets/summary.json") as f: 11 | test_data = json.load(f) 12 | test_result = test_data["statistic"]["total"] == test_data["statistic"]["passed"] 13 | 14 | coverage_result = float(ET.parse("./coverage.xml").getroot().attrib["line-rate"]) * 100.0 15 | 16 | if os.path.exists(badges_directory) and os.path.isdir(badges_directory): 17 | shutil.rmtree(badges_directory) 18 | os.mkdir(badges_directory) 19 | else: 20 | os.mkdir(badges_directory) 21 | 22 | url_data = "passing&color=brightgreen" if test_result else "failing&color=critical" 23 | response = requests.get("https://img.shields.io/static/v1?label=Tests&message=" + url_data) 24 | with open(badges_directory + "/tests.svg", "w") as f: 25 | f.write(response.text) 26 | url_data = "brightgreen" if coverage_result == 100.0 else "critical" 27 | response = requests.get(f"https://img.shields.io/static/v1?label=Coverage&message={coverage_result}%&color={url_data}") 28 | with open(badges_directory + "/coverage.svg", "w") as f: 29 | f.write(response.text) 30 | -------------------------------------------------------------------------------- /tests/test_docker_entrypoint.py: -------------------------------------------------------------------------------- 1 | """Tests for docker-entrypoint.sh script functionality.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | import os 6 | import subprocess 7 | import unittest 8 | 9 | 10 | class TestDockerEntrypoint(unittest.TestCase): 11 | """Tests class for docker-entrypoint.sh script.""" 12 | 13 | def setUp(self) -> None: 14 | """Initialize tests.""" 15 | self.entrypoint_path = os.path.join( 16 | os.path.dirname(os.path.dirname(__file__)), "docker-entrypoint.sh", 17 | ) 18 | return super().setUp() 19 | 20 | def test_entrypoint_script_exists(self): 21 | """Test that the docker-entrypoint.sh script exists.""" 22 | self.assertTrue(os.path.exists(self.entrypoint_path)) 23 | 24 | def test_entrypoint_script_syntax(self): 25 | """Test that the docker-entrypoint.sh script has valid shell syntax.""" 26 | result = subprocess.run( 27 | ["sh", "-n", self.entrypoint_path], 28 | capture_output=True, 29 | text=True, 30 | ) 31 | self.assertEqual(result.returncode, 0, f"Script syntax error: {result.stderr}") 32 | 33 | 34 | if __name__ == "__main__": 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /src/photo_cleanup_utils.py: -------------------------------------------------------------------------------- 1 | """Photo file cleanup utilities module. 2 | 3 | This module contains utilities for cleaning up obsolete photo files 4 | that are no longer on the server. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | from pathlib import Path 10 | 11 | from src import get_logger 12 | 13 | LOGGER = get_logger() 14 | 15 | 16 | def remove_obsolete_files(destination_path: str | None, tracked_files: set[str] | None) -> set[str]: 17 | """Remove local obsolete files that are no longer on server. 18 | 19 | Args: 20 | destination_path: Path to search for obsolete files 21 | tracked_files: Set of files that should be kept (files on server) 22 | 23 | Returns: 24 | Set of paths that were removed 25 | """ 26 | removed_paths = set() 27 | 28 | if not (destination_path and tracked_files is not None): 29 | return removed_paths 30 | 31 | for path in Path(destination_path).rglob("*"): 32 | local_file = str(path.absolute()) 33 | if local_file not in tracked_files: 34 | if path.is_file(): 35 | LOGGER.info(f"Removing {local_file} ...") 36 | path.unlink(missing_ok=True) 37 | removed_paths.add(local_file) 38 | 39 | return removed_paths 40 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Docker entrypoint script to replace s6-overlay functionality 3 | 4 | # Set default values 5 | PUID=${PUID:-911} 6 | PGID=${PGID:-911} 7 | 8 | # Update user and group IDs 9 | echo "Setting up user 'abc' with UID: $PUID, GID: $PGID" 10 | groupmod -o -g "$PGID" abc 11 | usermod -o -u "$PUID" abc 12 | 13 | # Display sponsorship message 14 | echo " 15 | ==================================================== 16 | To support this project, please consider sponsoring. 17 | https://github.com/sponsors/mandarons 18 | https://www.buymeacoffee.com/mandarons 19 | 20 | User UID: $(id -u abc) 21 | User GID: $(id -g abc) 22 | ====================================================" 23 | 24 | # Display build version if available 25 | if [ -f /build_version ]; then 26 | cat /build_version 27 | fi 28 | 29 | # Create necessary directories 30 | mkdir -p /icloud /config/session_data /home/abc 31 | 32 | # Set ownership if not already correct 33 | for dir in /app /config /icloud /home/abc; do 34 | if [ "$(stat -c %u:%g "$dir" 2>/dev/null)" != "$(id -u abc):$(id -g abc)" ]; then 35 | echo "Setting ownership for $dir" 36 | chown -R abc:abc "$dir" 37 | fi 38 | done 39 | 40 | # Execute the main application as abc user 41 | echo "Starting iCloud Docker application..." 42 | exec su-exec abc /app/init.sh 43 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests module.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | import os 6 | import uuid 7 | 8 | from ruamel.yaml import YAML 9 | 10 | from src import usage 11 | 12 | DATA_DIR = os.path.join(os.path.dirname(__file__), "data") 13 | CONFIG_PATH = os.path.join(DATA_DIR, "test_config.yaml") 14 | ENV_CONFIG_PATH = os.path.join(DATA_DIR, "test_config_env.yaml") 15 | TEMP_DIR = os.path.join(os.path.dirname(__file__), "temp") 16 | DRIVE_DIR = os.path.join(TEMP_DIR, "icloud", "drive") 17 | PHOTOS_DIR = os.path.join(TEMP_DIR, "icloud", "photos") 18 | 19 | 20 | def update_config(data): 21 | """Update config test config path.""" 22 | return YAML().dump(data=data, stream=open(file=CONFIG_PATH, mode="w", encoding="utf-8")) 23 | 24 | 25 | def mocked_usage_post(*args, **kwargs): 26 | """Mock the post method.""" 27 | 28 | class MockResponse: 29 | def __init__(self, json_data, status_code) -> None: 30 | self.json_data = json_data 31 | self.status_code = status_code 32 | self.ok = status_code == 201 33 | 34 | def json(self): 35 | return self.json_data 36 | 37 | if args[0] is usage.NEW_INSTALLATION_ENDPOINT: 38 | return MockResponse({"id": str(uuid.uuid4())}, 201) 39 | elif args[0] is usage.NEW_HEARTBEAT_ENDPOINT: 40 | return MockResponse({"message": "All good."}, 201) 41 | return MockResponse(None, 404) 42 | -------------------------------------------------------------------------------- /src/drive_cleanup.py: -------------------------------------------------------------------------------- 1 | """Drive cleanup utilities. 2 | 3 | This module provides cleanup functionality for removing obsolete files 4 | and directories during iCloud Drive sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | from pathlib import Path 10 | from shutil import rmtree 11 | 12 | from src import configure_icloudpy_logging, get_logger 13 | 14 | # Configure icloudpy logging immediately after import 15 | configure_icloudpy_logging() 16 | 17 | LOGGER = get_logger() 18 | 19 | 20 | def remove_obsolete(destination_path: str, files: set[str]) -> set[str]: 21 | """Remove local files and directories that no longer exist remotely. 22 | 23 | Args: 24 | destination_path: Root directory to clean up 25 | files: Set of file paths that should be kept (exist remotely) 26 | 27 | Returns: 28 | Set of paths that were removed 29 | """ 30 | removed_paths = set() 31 | if not (destination_path and files is not None): 32 | return removed_paths 33 | 34 | for path in Path(destination_path).rglob("*"): 35 | local_file = str(path.absolute()) 36 | if local_file not in files: 37 | LOGGER.info(f"Removing {local_file} ...") 38 | if path.is_file(): 39 | path.unlink(missing_ok=True) 40 | removed_paths.add(local_file) 41 | elif path.is_dir(): 42 | rmtree(local_file) 43 | removed_paths.add(local_file) 44 | return removed_paths 45 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | ignore: | 2 | azure-*.yml 3 | rules: 4 | braces: 5 | level: error 6 | min-spaces-inside: 0 7 | max-spaces-inside: 1 8 | min-spaces-inside-empty: -1 9 | max-spaces-inside-empty: -1 10 | brackets: 11 | level: error 12 | min-spaces-inside: 0 13 | max-spaces-inside: 0 14 | min-spaces-inside-empty: -1 15 | max-spaces-inside-empty: -1 16 | colons: 17 | level: error 18 | max-spaces-before: 0 19 | max-spaces-after: 1 20 | commas: 21 | level: error 22 | max-spaces-before: 0 23 | min-spaces-after: 1 24 | max-spaces-after: 1 25 | comments: 26 | level: error 27 | require-starting-space: true 28 | min-spaces-from-content: 1 29 | comments-indentation: 30 | level: error 31 | document-end: 32 | level: error 33 | present: false 34 | document-start: 35 | level: error 36 | present: false 37 | empty-lines: 38 | level: error 39 | max: 1 40 | max-start: 0 41 | max-end: 1 42 | hyphens: 43 | level: error 44 | max-spaces-after: 1 45 | indentation: 46 | level: error 47 | spaces: 2 48 | indent-sequences: true 49 | check-multi-line-strings: false 50 | key-duplicates: 51 | level: error 52 | line-length: disable 53 | new-line-at-end-of-file: 54 | level: error 55 | new-lines: 56 | level: error 57 | type: unix 58 | trailing-spaces: 59 | level: error 60 | truthy: 61 | level: error 62 | allowed-values: 63 | - "on" 64 | - "true" 65 | - "false" 66 | -------------------------------------------------------------------------------- /tests/test_container_integration.py: -------------------------------------------------------------------------------- 1 | """Integration tests for container functionality after s6-overlay removal.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | import os 6 | import subprocess 7 | import unittest 8 | 9 | 10 | class TestContainerIntegration(unittest.TestCase): 11 | """Integration tests for container functionality.""" 12 | 13 | def setUp(self) -> None: 14 | """Initialize tests.""" 15 | self.repo_root = os.path.dirname(os.path.dirname(__file__)) 16 | self.dockerfile_path = os.path.join(self.repo_root, "Dockerfile") 17 | self.init_script_path = os.path.join(self.repo_root, "init.sh") 18 | return super().setUp() 19 | 20 | def test_dockerfile_syntax(self): 21 | """Test that Dockerfile has valid syntax.""" 22 | # Basic syntax check by trying to parse it 23 | self.assertTrue(os.path.exists(self.dockerfile_path)) 24 | with open(self.dockerfile_path, encoding="utf-8") as f: 25 | content = f.read() 26 | # Check it starts with FROM 27 | self.assertTrue(content.strip().startswith("# syntax=docker/dockerfile:1")) 28 | 29 | def test_init_script_syntax(self): 30 | """Test that init.sh has valid shell syntax.""" 31 | result = subprocess.run( 32 | ["sh", "-n", self.init_script_path], 33 | capture_output=True, 34 | text=True, 35 | ) 36 | self.assertEqual(result.returncode, 0, f"Script syntax error: {result.stderr}") 37 | 38 | 39 | if __name__ == "__main__": 40 | unittest.main() 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2021, Mandar Patil (mandarons@pm.me) 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /tests/test_src_init.py: -------------------------------------------------------------------------------- 1 | """Tests for sync module.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | import logging 6 | import unittest 7 | from unittest.mock import patch 8 | 9 | import tests 10 | from src import get_logger, read_config 11 | 12 | 13 | class TestSrcInit(unittest.TestCase): 14 | """Tests class for sync module.""" 15 | 16 | def setUp(self) -> None: 17 | """Initialize tests.""" 18 | self.config = read_config(config_path=tests.CONFIG_PATH) 19 | return super().setUp() 20 | 21 | @patch("src.read_config") 22 | def test_get_logger_no_config(self, mock_read_config): 23 | """Test for no config.""" 24 | config = self.config.copy() 25 | # Add null handler if not configured 26 | del config["app"]["logger"] 27 | mock_read_config.return_value = config 28 | logger = get_logger() 29 | self.assertTrue(len([h for h in logger.handlers if isinstance(h, logging.NullHandler)]) > 0) 30 | 31 | @patch("src.read_config") 32 | def test_get_logger(self, mock_read_config): 33 | """Test for logger.""" 34 | config = self.config.copy() 35 | # success flow 36 | mock_read_config.return_value = config 37 | logger = get_logger() 38 | self.assertTrue(len(logger.handlers) > 1) 39 | 40 | @patch("src.read_config") 41 | def test_get_logger_no_duplicate_handlers(self, mock_read_config): 42 | """Test for no duplicate logger handlers.""" 43 | config = self.config.copy() 44 | # No duplicate handlers 45 | mock_read_config.return_value = config 46 | logger = get_logger() 47 | number_of_handlers = len(logger.handlers) 48 | logger = get_logger() 49 | self.assertEqual(len(logger.handlers), number_of_handlers) 50 | -------------------------------------------------------------------------------- /USAGE.md: -------------------------------------------------------------------------------- 1 | ### ICloud-Drive-Docker Usage Tracking 2 | 3 | We collect following information to analyze usage of `mandarons/icloud-drive-docker` project: 4 | 5 | 1. `Application version` - to track which versions of `mandarons/icloud-drive-docker` are currently in use 6 | 2. `Sync statistics` - anonymized aggregated data about sync operations (file counts, sync duration, error indicators) 7 | 3. `Installation events` - new installations and upgrades for usage analytics 8 | 9 | On server side, this project uses `IP address` to determine `country` of `mandarons/icloud-drive-docker` installation. 10 | 11 | Collecting this data helps keep supporting this project and drive future improvements. **No personally identifiable information is collected.** Aggregate data is made available at [Wapar](https://wapar.mandarons.com). 12 | 13 | ## How to Opt Out 14 | 15 | You can completely disable usage tracking by adding the following to your `config.yaml`: 16 | 17 | ```yaml 18 | app: 19 | usage_tracking: 20 | enabled: false 21 | ``` 22 | 23 | When disabled, no usage data will be collected or transmitted. The sync functionality remains completely unaffected. 24 | 25 | ## Data Collected 26 | 27 | ### Installation Data 28 | - Application version 29 | - Installation ID (randomly generated UUID) 30 | - Country (derived from IP address on server side) 31 | 32 | ### Sync Statistics (when available) 33 | - Sync duration 34 | - Number of files/photos processed (counts only) 35 | - Data transfer volumes (bytes) 36 | - Error indicators (boolean flags, no error details) 37 | - Timestamp of sync operation 38 | 39 | ### Privacy Guarantees 40 | - No file names, paths, or content 41 | - No personal information or account details 42 | - No iCloud credentials or tokens 43 | - Data is aggregated and anonymized 44 | - Opt-out available at any time 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | # Build stage 4 | FROM python:3.10-alpine3.22 AS builder 5 | 6 | # Install build dependencies 7 | COPY requirements.txt . 8 | RUN \ 9 | echo "**** install build packages ****" && \ 10 | apk add --no-cache --virtual=build-dependencies \ 11 | git \ 12 | gcc \ 13 | musl-dev \ 14 | python3-dev \ 15 | libffi-dev \ 16 | openssl-dev \ 17 | cargo && \ 18 | echo "**** install icloud app ****" && \ 19 | pip install -U --no-cache-dir \ 20 | pip \ 21 | wheel && \ 22 | pip install -U --no-cache-dir -r requirements.txt 23 | 24 | # Runtime stage 25 | FROM python:3.10-alpine3.22 26 | 27 | # set version label 28 | ARG APP_VERSION=dev 29 | ARG NEW_INSTALLATION_ENDPOINT=dev 30 | ARG NEW_HEARTBEAT_ENDPOINT=dev 31 | ENV NEW_INSTALLATION_ENDPOINT=$NEW_INSTALLATION_ENDPOINT 32 | ENV NEW_HEARTBEAT_ENDPOINT=$NEW_HEARTBEAT_ENDPOINT 33 | ENV APP_VERSION=$APP_VERSION 34 | LABEL maintainer="mandarons" 35 | 36 | # Set environment variables 37 | ENV HOME="/app" 38 | ENV PUID=911 39 | ENV PGID=911 40 | 41 | # Install runtime dependencies and create user 42 | RUN \ 43 | echo "**** update package repository ****" && \ 44 | apk update && \ 45 | echo "**** install runtime packages ****" && \ 46 | apk add --no-cache \ 47 | libmagic \ 48 | shadow \ 49 | su-exec && \ 50 | echo "**** create user ****" && \ 51 | addgroup -g 911 abc && \ 52 | adduser -D -u 911 -G abc -h /home/abc -s /bin/sh abc 53 | 54 | # Copy Python packages from builder stage 55 | COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages 56 | COPY --from=builder /usr/local/bin /usr/local/bin 57 | 58 | # add local files 59 | COPY . /app/ 60 | WORKDIR /app 61 | 62 | # Create entrypoint script 63 | COPY docker-entrypoint.sh /usr/local/bin/ 64 | RUN chmod +x /usr/local/bin/docker-entrypoint.sh 65 | 66 | EXPOSE 80 67 | CMD ["/usr/local/bin/docker-entrypoint.sh"] 68 | -------------------------------------------------------------------------------- /src/drive_folder_processing.py: -------------------------------------------------------------------------------- 1 | """Folder processing utilities. 2 | 3 | This module provides folder creation and processing functionality for 4 | iCloud Drive sync operations, separating folder logic from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | import unicodedata 11 | from typing import Any 12 | from urllib.parse import unquote 13 | 14 | from src import configure_icloudpy_logging, get_logger 15 | from src.drive_filtering import wanted_folder 16 | 17 | # Configure icloudpy logging immediately after import 18 | configure_icloudpy_logging() 19 | 20 | LOGGER = get_logger() 21 | 22 | 23 | def process_folder( 24 | item: Any, 25 | destination_path: str, 26 | filters: list[str] | None, 27 | ignore: list[str] | None, 28 | root: str, 29 | ) -> str | None: 30 | """Process a folder item by creating the local directory if wanted. 31 | 32 | Args: 33 | item: iCloud folder item 34 | destination_path: Local destination directory 35 | filters: Folder filters to apply 36 | ignore: Ignore patterns 37 | root: Root directory for relative path calculations 38 | 39 | Returns: 40 | Path to the created directory, or None if folder should be skipped 41 | """ 42 | if not (item and destination_path and root): 43 | return None 44 | 45 | # Decode URL-encoded folder name from iCloud API 46 | # This handles special characters like %CC%88 (combining diacritical marks) 47 | decoded_name = unquote(item.name) 48 | new_directory = os.path.join(destination_path, decoded_name) 49 | new_directory_norm = unicodedata.normalize("NFC", new_directory) 50 | 51 | if not wanted_folder(filters=filters, ignore=ignore, folder_path=new_directory_norm, root=root): 52 | LOGGER.debug(f"Skipping the unwanted folder {new_directory} ...") 53 | return None 54 | 55 | os.makedirs(new_directory_norm, exist_ok=True) 56 | return new_directory 57 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "iCloud Drive Docker - Dev", 3 | "context": "..", 4 | "dockerFile": "Dockerfile", 5 | "containerEnv": { 6 | "DEVCONTAINER": "1" 7 | }, 8 | "features": { 9 | "ghcr.io/devcontainers/features/github-cli:1": {} 10 | }, 11 | "customizations": { 12 | "vscode": { 13 | "extensions": [ 14 | "ms-python.vscode-pylance", 15 | "visualstudioexptteam.vscodeintellicode", 16 | "redhat.vscode-yaml", 17 | "esbenp.prettier-vscode", 18 | "GitHub.vscode-pull-request-github", 19 | "github.vscode-github-actions", 20 | "charliermarsh.ruff" 21 | ], 22 | "settings": { 23 | "[python]": { 24 | "diffEditor.ignoreTrimWhitespace": false, 25 | "editor.formatOnType": true, 26 | "editor.formatOnSave": true, 27 | "editor.wordBasedSuggestions": "off", 28 | "editor.defaultFormatter": "charliermarsh.ruff", 29 | "editor.codeActionsOnSave": { 30 | "source.fixAll": "explicit", 31 | "source.organizeImports": "explicit" 32 | } 33 | }, 34 | "python.pythonPath": "./venv/bin/python", 35 | "python.testing.pytestArgs": ["--no-cov"], 36 | "files.trimTrailingWhitespace": true, 37 | "terminal.integrated.defaultProfile.linux": "bash", 38 | "yaml.customTags": [ 39 | "!input scalar", 40 | "!secret scalar", 41 | "!include_dir_named scalar", 42 | "!include_dir_list scalar", 43 | "!include_dir_merge_list scalar", 44 | "!include_dir_merge_named scalar" 45 | ] 46 | } 47 | } 48 | }, 49 | "remoteUser": "vscode", 50 | "postCreateCommand": "uv venv && . .venv/bin/activate && uv pip install -r requirements-test.txt && git config commit.gpgsign true", 51 | "mounts": [ 52 | "source=${localEnv:HOME}${localEnv:USERPROFILE}/.gnupg,target=/home/vscode/.gnupg,type=bind,consistency=cached", 53 | "source=${localWorkspaceFolder}/.cache,target=/home/vscode/.cache,type=bind,consistency=cached", 54 | "source=${localWorkspaceFolder}/.vscode-server,target=/home/vscode/.vscode-server,type=bind,consistency=cached" 55 | ] 56 | } -------------------------------------------------------------------------------- /src/config_logging.py: -------------------------------------------------------------------------------- 1 | """Logging utilities for configuration-related operations. 2 | 3 | Provides reusable logging functions to separate logging concerns from 4 | config retrieval logic, following Single Responsibility Principle. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | from typing import Any 10 | 11 | from src import get_logger 12 | 13 | LOGGER = get_logger() 14 | 15 | 16 | def log_config_not_found_warning(config_path: list[str], message: str) -> None: 17 | """Log a warning when a config path is not found. 18 | 19 | Args: 20 | config_path: List of config keys forming the path 21 | message: Custom warning message to log 22 | """ 23 | from src.config_utils import config_path_to_string 24 | 25 | path_str = config_path_to_string(config_path) 26 | LOGGER.warning(f"{path_str} {message}") 27 | 28 | 29 | def log_config_found_info(message: str) -> None: 30 | """Log an info message when config value is found and processed. 31 | 32 | Args: 33 | message: Info message to log 34 | """ 35 | LOGGER.info(message) 36 | 37 | 38 | def log_config_debug(message: str) -> None: 39 | """Log a debug message for config processing. 40 | 41 | Args: 42 | message: Debug message to log 43 | """ 44 | LOGGER.debug(message) 45 | 46 | 47 | def log_config_error(config_path: list[str], message: str) -> None: 48 | """Log an error for config validation issues. 49 | 50 | Args: 51 | config_path: List of config keys forming the path 52 | message: Error message to log 53 | """ 54 | from src.config_utils import config_path_to_string 55 | 56 | path_str = config_path_to_string(config_path) 57 | LOGGER.error(f"{path_str}: {message}") 58 | 59 | 60 | def log_invalid_config_value(config_path: list[str], invalid_value: Any, valid_values: str) -> None: 61 | """Log warning about invalid config value. 62 | 63 | Args: 64 | config_path: List of config keys forming the path 65 | invalid_value: The invalid value that was found 66 | valid_values: Description of valid values 67 | """ 68 | from src.config_utils import config_path_to_string 69 | 70 | path_str = config_path_to_string(config_path) 71 | LOGGER.warning(f"Invalid value '{invalid_value}' at {path_str}. Valid values: {valid_values}") 72 | -------------------------------------------------------------------------------- /src/hardlink_registry.py: -------------------------------------------------------------------------------- 1 | """Hardlink registry management module. 2 | 3 | This module contains utilities for managing hardlink registry 4 | during photo synchronization to avoid duplicate downloads. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | 10 | from src import get_logger 11 | 12 | LOGGER = get_logger() 13 | 14 | 15 | class HardlinkRegistry: 16 | """Registry to track downloaded photos for hardlink creation. 17 | 18 | This class manages a registry of downloaded photos to enable hardlink 19 | creation for duplicate photos across different albums. 20 | """ 21 | 22 | def __init__(self): 23 | """Initialize hardlink registry.""" 24 | self._registry: dict[str, str] = {} 25 | 26 | def get_existing_path(self, photo_id: str, file_size: str) -> str | None: 27 | """Get existing path for photo if it was already downloaded. 28 | 29 | Args: 30 | photo_id: Unique photo identifier 31 | file_size: File size variant (original, medium, thumb, etc.) 32 | 33 | Returns: 34 | Path to existing file if found, None otherwise 35 | """ 36 | photo_key = f"{photo_id}_{file_size}" 37 | return self._registry.get(photo_key) 38 | 39 | def register_photo_path(self, photo_id: str, file_size: str, file_path: str) -> None: 40 | """Register a downloaded photo path for future hardlink creation. 41 | 42 | Args: 43 | photo_id: Unique photo identifier 44 | file_size: File size variant (original, medium, thumb, etc.) 45 | file_path: Path where photo was downloaded 46 | """ 47 | photo_key = f"{photo_id}_{file_size}" 48 | self._registry[photo_key] = file_path 49 | 50 | def get_registry_size(self) -> int: 51 | """Get number of registered photos. 52 | 53 | Returns: 54 | Number of photos in the registry 55 | """ 56 | return len(self._registry) 57 | 58 | def clear(self) -> None: 59 | """Clear the registry.""" 60 | self._registry.clear() 61 | 62 | 63 | def create_hardlink_registry(use_hardlinks: bool) -> HardlinkRegistry | None: 64 | """Create hardlink registry if hardlinks are enabled. 65 | 66 | Args: 67 | use_hardlinks: Whether hardlinks are enabled in configuration 68 | 69 | Returns: 70 | HardlinkRegistry instance if hardlinks enabled, None otherwise 71 | """ 72 | return HardlinkRegistry() if use_hardlinks else None 73 | -------------------------------------------------------------------------------- /src/drive_file_download.py: -------------------------------------------------------------------------------- 1 | """File download utilities. 2 | 3 | This module provides file downloading functionality for iCloud Drive sync, 4 | separating download logic from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | import time 11 | from typing import Any 12 | 13 | from src import configure_icloudpy_logging, get_logger 14 | from src.drive_package_processing import process_package 15 | 16 | # Configure icloudpy logging immediately after import 17 | configure_icloudpy_logging() 18 | 19 | LOGGER = get_logger() 20 | 21 | 22 | def download_file(item: Any, local_file: str) -> str | None: 23 | """Download a file from iCloud to local filesystem. 24 | 25 | This function handles the actual download of files from iCloud, including 26 | package detection and processing, and sets the correct modification time. 27 | 28 | Args: 29 | item: iCloud file item to download 30 | local_file: Local path to save the file 31 | 32 | Returns: 33 | Path to the downloaded/processed file, or None if download failed 34 | """ 35 | if not (item and local_file): 36 | return None 37 | 38 | LOGGER.info(f"Downloading {local_file} ...") 39 | try: 40 | with item.open(stream=True) as response: 41 | with open(local_file, "wb") as file_out: 42 | for chunk in response.iter_content(4 * 1024 * 1024): 43 | file_out.write(chunk) 44 | 45 | # Check if this is a package that needs processing 46 | if response.url and "/packageDownload?" in response.url: 47 | processed_file = process_package(local_file=local_file) 48 | if processed_file: 49 | local_file = processed_file 50 | else: 51 | return None 52 | 53 | # Set the file modification time to match the remote file 54 | item_modified_time = time.mktime(item.date_modified.timetuple()) 55 | os.utime(local_file, (item_modified_time, item_modified_time)) 56 | 57 | except Exception as e: 58 | # Enhanced error logging with file path context 59 | # This catches all exceptions including iCloudPy errors like ObjectNotFoundException 60 | error_msg = str(e) 61 | if "ObjectNotFoundException" in error_msg or "NOT_FOUND" in error_msg: 62 | LOGGER.error(f"File not found in iCloud Drive - {local_file}: {error_msg}") 63 | else: 64 | LOGGER.error(f"Failed to download {local_file}: {error_msg}") 65 | return None 66 | 67 | return local_file 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | .vscode-server 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # pipenv 89 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 90 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 91 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 92 | # install all needed dependencies. 93 | #Pipfile.lock 94 | 95 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Other 132 | src/drive 133 | allure-results 134 | allure-report 135 | .history 136 | ignore-config.yaml 137 | session 138 | session_data 139 | icloud/ 140 | .ruff_cache/ -------------------------------------------------------------------------------- /tests/data/test_config_env.yaml: -------------------------------------------------------------------------------- 1 | app: 2 | logger: 3 | # level - debug, info (default), warning or error 4 | level: debug 5 | # log filename icloud.log (default) 6 | filename: config_loaded_using_env_config_path.log 7 | credentials: 8 | # iCloud drive username 9 | username: user@test.com 10 | # Retry failed login (with/without 2FA) every 10 minutes 11 | retry_login_interval: 600 12 | # Drive destination 13 | root: "./icloud" 14 | notifications: 15 | sync_summary: 16 | enabled: false 17 | on_success: true 18 | on_error: true 19 | min_downloads: 10 20 | smtp: 21 | # If you want to recieve email notifications about expired/missing 2FA credentials then uncomment 22 | # email: sender@test.com 23 | # Uncomment this if your SMTP username is different than your sender address (for services like AWS SES) 24 | # username: "" 25 | # default to is same as email above 26 | # to: receiver@test.com 27 | # password: 28 | # host: smtp.test.com 29 | # port: 587 30 | # If your email provider doesn't handle TLS 31 | # no_tls: true 32 | # valid values are - global (default - uses .com) or china (uses .com.cn) 33 | region: global 34 | 35 | drive: 36 | destination: "./drive" 37 | remove_obsolete: true 38 | sync_interval: -1 39 | ignore: 40 | - "*.psd" 41 | - .git/ 42 | filters: 43 | # File filters to be included in syncing iCloud drive content 44 | folders: 45 | - dir1/dir2/dir3 46 | - Keynote 47 | - icloudpy 48 | - Obsidian 49 | file_extensions: 50 | # File extensions to be included 51 | - pdf 52 | - png 53 | - jpg 54 | - jpeg 55 | - md 56 | - band 57 | - xmcf 58 | photos: 59 | destination: photos 60 | remove_obsolete: false 61 | sync_interval: -1 62 | all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem 63 | use_hardlinks: false # Optional, default false. If true and all_albums is true, create hard links for duplicate photos instead of separate copies. Saves storage space. 64 | # folder_format: "%Y/%m" # optional, if set put photos in subfolders according to format. Format cheatsheet - https://strftime.org 65 | filters: 66 | # if all_albums is false - albums list is used as filter-in, if all_albums is true - albums list is used as filter-out 67 | # if albums list is empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders 68 | albums: 69 | - "album 2" 70 | - album-1 71 | file_sizes: 72 | # Valid values are [original, medium or thumb] 73 | - original 74 | - medium 75 | - thumb 76 | -------------------------------------------------------------------------------- /unraid/icloud.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | icloud 4 | mandarons/icloud-drive 5 | https://hub.docker.com/r/mandarons/icloud-drive 6 | bridge 7 | 8 | sh 9 | false 10 | https://forums.unraid.net/topic/168782-mandarons-icloud/ 11 | https://github.com/mandarons/icloud-drive-docker 12 | Dockerized iCloud Client - make a local copy of your iCloud documents and photos, and keep it automatically up-to-date. 13 | Backup: Cloud: Downloaders: HomeAutomation: Tools: Status:Stable 14 | 15 | https://raw.githubusercontent.com/mandarons/icloud-drive-docker/main/unraid/icloud.xml 16 | https://help.apple.com/assets/60AD31069883FC55AC222539/60AD310B9883FC55AC22254A/de_DE/712e44cf3701cf5bc9580c9367fa5526.png 17 | 18 | 19 | 20 | 1719337818 21 | To support this project, please consider sponsoring. 22 | https://github.com/sponsors/mandarons 23 | To get started, 24 | 1. Create 'config' folder and put 'config.yaml' file (sample file available at: https://raw.githubusercontent.com/mandarons/icloud-drive-docker/main/config.yaml) 25 | 2. Update the `config/config.yaml` file per your needs 26 | 3. Create 'icloud' folder to be used as local copy of your icloud (Drive and Photos) 27 | 99 28 | 100 29 | /mnt/user/appdata/icloud/config 30 | <provide location to icloud backup> 31 | <your icloud.com password> 32 | /config/config.yaml 33 | 34 | -------------------------------------------------------------------------------- /.ruff.toml: -------------------------------------------------------------------------------- 1 | exclude = [ 2 | ".bzr", 3 | ".direnv", 4 | ".eggs", 5 | ".git", 6 | ".git-rewrite", 7 | ".hg", 8 | ".ipynb_checkpoints", 9 | ".mypy_cache", 10 | ".nox", 11 | ".pants.d", 12 | ".pyenv", 13 | ".pytest_cache", 14 | ".pytype", 15 | ".ruff_cache", 16 | ".svn", 17 | ".tox", 18 | ".venv", 19 | ".vscode", 20 | "__pypackages__", 21 | "_build", 22 | "buck-out", 23 | "build", 24 | "dist", 25 | "node_modules", 26 | "site-packages", 27 | "venv", 28 | ] 29 | 30 | # Same as Black. 31 | line-length = 120 32 | indent-width = 4 33 | 34 | # Assume Python 3.8 35 | # target-version = "py38" 36 | 37 | [lint] 38 | # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. 39 | # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or 40 | # McCabe complexity (`C901`) by default. 41 | select = ["F", "E", "W", 42 | # "C90", 43 | "I", 44 | # "N", 45 | # "D", 46 | "UP", "YTT", 47 | # "ANN", 48 | "ASYNC", 49 | # "S", 50 | # "BLE", 51 | # "FBT", 52 | # "B", 53 | # "A", 54 | "COM", "C4", 55 | # "DTZ", 56 | # "T10", 57 | "DJ", "EM", "EXE", 58 | #"FA", 59 | # "ISC", 60 | "ICN", 61 | # "LOG", 62 | # "G", 63 | "INP", "PIE", 64 | # "T20", 65 | "PYI", 66 | # "PT", 67 | "Q", "RSE", 68 | #"RET", 69 | "SLF", "SLOT", 70 | # "SIM", 71 | "TID", "TCH", "INT", 72 | # "ARG", 73 | # "PTH", 74 | "TD", 75 | "FIX", 76 | # "ERA", 77 | "PD", "PGH", 78 | # "PL", 79 | # "TRY", 80 | # "FLY", 81 | "NPY", "AIR", "PERF", 82 | # "FURB", 83 | # "RUF" 84 | ] 85 | ignore = ["E501"] 86 | 87 | # Allow fix for all enabled rules (when `--fix`) is provided. 88 | fixable = ["ALL"] 89 | unfixable = ["B"] 90 | 91 | # Allow unused variables when underscore-prefixed. 92 | dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" 93 | # 4. Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. 94 | [lint.per-file-ignores] 95 | "__init__.py" = ["E402"] 96 | "**/{tests,docs,tools}/*" = ["E402"] 97 | 98 | 99 | [format] 100 | # Like Black, use double quotes for strings. 101 | quote-style = "double" 102 | 103 | # Like Black, indent with spaces, rather than tabs. 104 | indent-style = "space" 105 | 106 | # Like Black, respect magic trailing commas. 107 | skip-magic-trailing-comma = false 108 | 109 | # Like Black, automatically detect the appropriate line ending. 110 | line-ending = "auto" 111 | 112 | # Enable auto-formatting of code examples in docstrings. Markdown, 113 | # reStructuredText code/literal blocks and doctests are all supported. 114 | # 115 | # This is currently disabled by default, but it is planned for this 116 | # to be opt-out in the future. 117 | # docstring-code-format = true 118 | # docstring-code-line-length = 120 119 | 120 | # Set the line length limit used when formatting code snippets in 121 | # docstrings. 122 | # 123 | # This only has an effect when the `docstring-code-format` setting is 124 | # enabled. 125 | # docstring-code-line-length = "dynamic" -------------------------------------------------------------------------------- /.github/workflows/official-release-to-docker-hub.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Release to Docker Hub 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | tags: 10 | - "v*.*.*" 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "build" 18 | build: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | 22 | # Steps represent a sequence of tasks that will be executed as part of the job 23 | steps: 24 | - name: Check Out Repo 25 | uses: actions/checkout@v4 26 | - name: Prepare 27 | id: prep 28 | run: | 29 | DOCKER_IMAGE=mandarons/icloud-drive 30 | if [[ $GITHUB_REF == refs/tags/* ]]; then 31 | VERSION=${GITHUB_REF#refs/tags/v} 32 | fi 33 | TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:latest" 34 | echo "${TAGS}" 35 | echo "tags=${TAGS}" >> $GITHUB_OUTPUT 36 | echo "version=${VERSION}" >> $GITHUB_OUTPUT 37 | - name: Cache Docker layers 38 | uses: actions/cache@v4 39 | with: 40 | path: /tmp/.buildx-cache 41 | key: ${{ runner.os }}-buildx-${{ github.sha }} 42 | restore-keys: | 43 | ${{ runner.os }}-buildx- 44 | - name: Login to Docker Hub 45 | uses: docker/login-action@v3 46 | with: 47 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 48 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 49 | - name: Set up Docker QEMU 50 | uses: docker/setup-qemu-action@v3 51 | - name: Set up Docker Buildx 52 | id: buildx 53 | uses: docker/setup-buildx-action@v3 54 | - name: Build and push 55 | id: docker_build 56 | uses: docker/build-push-action@v5 57 | with: 58 | context: ./ 59 | file: ./Dockerfile 60 | build-args: | 61 | NEW_INSTALLATION_ENDPOINT=${{ secrets.NEW_INSTALLATION_ENDPOINT }} 62 | NEW_HEARTBEAT_ENDPOINT=${{ secrets.NEW_HEARTBEAT_ENDPOINT }} 63 | APP_VERSION=${{ steps.prep.outputs.version }} 64 | push: true 65 | platforms: linux/amd64, linux/arm64 66 | tags: ${{ steps.prep.outputs.tags }} 67 | cache-from: type=local,src=/tmp/.buildx-cache 68 | cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max 69 | - # Temp fix 70 | # https://github.com/docker/build-push-action/issues/252 71 | # https://github.com/moby/buildkit/issues/1896 72 | name: Move cache 73 | run: | 74 | rm -rf /tmp/.buildx-cache 75 | mv /tmp/.buildx-cache-new /tmp/.buildx-cache 76 | - name: Image digest 77 | run: echo ${{ steps.docker_build.outputs.digest }} 78 | -------------------------------------------------------------------------------- /tests/data/test_config.yaml: -------------------------------------------------------------------------------- 1 | app: 2 | logger: 3 | # level - debug, info (default), warning or error 4 | level: debug 5 | # log filename icloud.log (default) 6 | filename: icloud.log 7 | credentials: 8 | # iCloud drive username 9 | username: user@test.com 10 | # Retry failed login (with/without 2FA) every 10 minutes 11 | retry_login_interval: 600 12 | # Drive destination 13 | root: "./icloud" 14 | notifications: 15 | sync_summary: 16 | # enabled: false (commented out to test default behavior) 17 | on_success: true 18 | on_error: true 19 | min_downloads: 1 20 | discord: 21 | # webhook_url: 22 | # username: icloud-docker 23 | pushover: 24 | # user_key: 25 | # api_token: 26 | smtp: 27 | # If you want to recieve email notifications about expired/missing 2FA credentials then uncomment 28 | # email: sender@test.com 29 | # Uncomment this if your SMTP username is different than your sender address (for services like AWS SES) 30 | # username: "" 31 | # default to is same as email above 32 | # to: receiver@test.com 33 | # password: 34 | # host: smtp.test.com 35 | # port: 587 36 | # If your email provider doesn't handle TLS 37 | # no_tls: true 38 | # valid values are - global (default - uses .com) or china (uses .com.cn) 39 | region: global 40 | 41 | drive: 42 | destination: "./drive" 43 | # Remove local files that are not present on server (i.e. files delete on server) 44 | remove_obsolete: true 45 | sync_interval: -1 46 | ignore: 47 | - "*.psd" 48 | - .git/ 49 | filters: 50 | # List of libraries to download. If omitted (default), photos from all libraries (own and shared) are downloaded. If included, photos only 51 | # from the listed libraries are downloaded. 52 | # libraries: 53 | # - PrimarySync # Name of the own library 54 | 55 | # File filters to be included in syncing iCloud drive content 56 | folders: 57 | - dir1/dir2/dir3 58 | - Keynote 59 | - icloudpy 60 | - Obsidian 61 | file_extensions: 62 | # File extensions to be included 63 | - pdf 64 | - png 65 | - jpg 66 | - jpeg 67 | - md 68 | - band 69 | - xmcf 70 | photos: 71 | destination: photos 72 | # Remove local photos that are not present on server (i.e. photos delete on server) 73 | remove_obsolete: false 74 | sync_interval: -1 75 | all_albums: false # Set back to default for existing tests 76 | use_hardlinks: false # Set to default for existing tests 77 | # folder_format: "%Y/%m" # optional, if set put photos in subfolders according to format. Format cheatsheet - https://strftime.org 78 | filters: 79 | libraries: # Optional, specify list of libraries to download photos from 80 | - PrimarySync # Library of the user 81 | # - SharedSync-abcd # Library of another user 82 | # if all_albums is false - albums list is used as filter-in, if all_albums is true - albums list is used as filter-out 83 | # if albums list is empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders 84 | albums: 85 | - "album 2" 86 | - album-1 87 | file_sizes: 88 | # Valid values are [original, medium or thumb] 89 | - original 90 | - medium 91 | - thumb 92 | -------------------------------------------------------------------------------- /src/drive_package_processing.py: -------------------------------------------------------------------------------- 1 | """Package processing utilities. 2 | 3 | This module provides package extraction and processing functionality, 4 | separating archive handling logic from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import gzip 10 | import os 11 | import unicodedata 12 | import zipfile 13 | from shutil import copyfileobj 14 | 15 | import magic 16 | 17 | from src import configure_icloudpy_logging, get_logger 18 | 19 | # Configure icloudpy logging immediately after import 20 | configure_icloudpy_logging() 21 | 22 | LOGGER = get_logger() 23 | 24 | 25 | def process_package(local_file: str) -> str | None: 26 | """Process and extract a downloaded package file. 27 | 28 | This function handles different archive types (ZIP, gzip) and extracts them 29 | to the appropriate location. It also handles Unicode normalization for 30 | cross-platform compatibility. 31 | 32 | Args: 33 | local_file: Path to the downloaded package file 34 | 35 | Returns: 36 | Path to the processed file/directory, or False if processing failed 37 | """ 38 | archive_file = local_file 39 | magic_object = magic.Magic(mime=True) 40 | file_mime_type = magic_object.from_file(filename=local_file) 41 | 42 | if file_mime_type == "application/zip": 43 | return _process_zip_package(local_file, archive_file) 44 | elif file_mime_type == "application/gzip": 45 | return _process_gzip_package(local_file, archive_file) 46 | else: 47 | LOGGER.error( 48 | f"Unhandled file type - cannot unpack the package {file_mime_type}.", 49 | ) 50 | return None 51 | 52 | 53 | def _process_zip_package(local_file: str, archive_file: str) -> str: 54 | """Process a ZIP package file. 55 | 56 | Args: 57 | local_file: Original file path 58 | archive_file: Archive file path 59 | 60 | Returns: 61 | Path to the processed file 62 | """ 63 | archive_file += ".zip" 64 | os.rename(local_file, archive_file) 65 | LOGGER.info(f"Unpacking {archive_file} to {os.path.dirname(archive_file)}") 66 | zipfile.ZipFile(archive_file).extractall(path=os.path.dirname(archive_file)) 67 | 68 | # Handle Unicode normalization for cross-platform compatibility 69 | normalized_path = unicodedata.normalize("NFD", local_file) 70 | if normalized_path != local_file: 71 | os.rename(local_file, normalized_path) 72 | local_file = normalized_path 73 | 74 | os.remove(archive_file) 75 | LOGGER.info(f"Successfully unpacked the package {archive_file}.") 76 | return local_file 77 | 78 | 79 | def _process_gzip_package(local_file: str, archive_file: str) -> str | None: 80 | """Process a gzip package file. 81 | 82 | Args: 83 | local_file: Original file path 84 | archive_file: Archive file path 85 | 86 | Returns: 87 | Path to the processed file, or None if processing failed 88 | """ 89 | archive_file += ".gz" 90 | os.rename(local_file, archive_file) 91 | LOGGER.info(f"Unpacking {archive_file} to {os.path.dirname(local_file)}") 92 | 93 | with gzip.GzipFile(filename=archive_file, mode="rb") as gz_file: 94 | with open(file=local_file, mode="wb") as package_file: 95 | copyfileobj(gz_file, package_file) 96 | 97 | os.remove(archive_file) 98 | 99 | # Recursively process the extracted file (might be another archive) 100 | return process_package(local_file=local_file) 101 | -------------------------------------------------------------------------------- /src/config_utils.py: -------------------------------------------------------------------------------- 1 | """Configuration utility functions for reusable config operations. 2 | 3 | This module provides low-level utilities for configuration traversal and retrieval, 4 | separated from business logic to follow Single Responsibility Principle. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | from typing import Any 10 | 11 | 12 | def config_path_to_string(config_path: list[str]) -> str: 13 | """Build config path as string for display purposes. 14 | 15 | Args: 16 | config_path: List of config keys forming a path (e.g., ["app", "credentials", "username"]) 17 | 18 | Returns: 19 | String representation of the config path (e.g., "app > credentials > username") 20 | """ 21 | return " > ".join(config_path) 22 | 23 | 24 | def traverse_config_path(config: dict, config_path: list[str]) -> bool: 25 | """Traverse and validate existence of a config path. 26 | 27 | Recursively checks if a path exists in the configuration dictionary. 28 | Does not retrieve values, only validates path existence. 29 | 30 | Args: 31 | config: Configuration dictionary to traverse 32 | config_path: List of keys forming the path to check 33 | 34 | Returns: 35 | True if path exists and is valid, False otherwise 36 | """ 37 | if len(config_path) == 0: 38 | return True 39 | if not (config and config_path[0] in config): 40 | return False 41 | return traverse_config_path(config[config_path[0]], config_path=config_path[1:]) 42 | 43 | 44 | def get_config_value(config: dict, config_path: list[str]) -> Any: 45 | """Retrieve value from config using a path. 46 | 47 | Recursively navigates the configuration dictionary to retrieve a value. 48 | Should only be called after validating path existence with traverse_config_path(). 49 | 50 | Args: 51 | config: Configuration dictionary 52 | config_path: List of keys forming the path to the value 53 | 54 | Returns: 55 | The configuration value at the specified path 56 | 57 | Raises: 58 | KeyError: If the path doesn't exist (should be prevented by prior validation) 59 | """ 60 | if len(config_path) == 1: 61 | return config[config_path[0]] 62 | return get_config_value(config=config[config_path[0]], config_path=config_path[1:]) 63 | 64 | 65 | def get_config_value_or_none(config: dict, config_path: list[str]) -> Any | None: 66 | """Safely retrieve config value or return None if path doesn't exist. 67 | 68 | Combines path validation and value retrieval for cases where None is acceptable. 69 | 70 | Args: 71 | config: Configuration dictionary 72 | config_path: List of keys forming the path to the value 73 | 74 | Returns: 75 | The configuration value if path exists, None otherwise 76 | """ 77 | if not traverse_config_path(config=config, config_path=config_path): 78 | return None 79 | return get_config_value(config=config, config_path=config_path) 80 | 81 | 82 | def get_config_value_or_default(config: dict, config_path: list[str], default: Any) -> Any: 83 | """Retrieve config value or return default if path doesn't exist. 84 | 85 | Args: 86 | config: Configuration dictionary 87 | config_path: List of keys forming the path to the value 88 | default: Default value to return if path doesn't exist 89 | 90 | Returns: 91 | The configuration value if path exists, default otherwise 92 | """ 93 | value = get_config_value_or_none(config=config, config_path=config_path) 94 | return value if value is not None else default 95 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | app: 2 | logger: 3 | # level - debug, info (default), warning or error 4 | level: "info" 5 | # log filename icloud.log 6 | filename: "/config/icloud.log" 7 | credentials: 8 | # iCloud drive username 9 | username: "please@replace.me" 10 | # Retry login interval - default is 10 minutes 11 | retry_login_interval: 600 12 | # Drive destination 13 | root: "/icloud" 14 | # Maximum number of parallel download threads for both drive and photos 15 | # auto: automatically set based on CPU cores (default, max 8) 16 | # integer: specific number of threads (max 16) 17 | # max_threads: auto 18 | # max_threads: 4 19 | notifications: 20 | # Sync summary notifications - sent after each sync cycle with statistics 21 | sync_summary: 22 | # Enable/disable sync summary notifications (default: false) 23 | enabled: false 24 | # Send notifications on successful syncs (default: true when enabled) 25 | on_success: true 26 | # Send notifications when errors occur during sync (default: true when enabled) 27 | on_error: true 28 | # Minimum number of downloads required to send notification (default: 1) 29 | # Set to 0 to always send notifications regardless of download count 30 | min_downloads: 1 31 | discord: 32 | # webhook_url: 33 | # username: icloud-docker #or any other name you prefer 34 | telegram: 35 | # bot_token: 36 | # chat_id: 37 | pushover: 38 | # user_key: 39 | # api_token: 40 | smtp: 41 | # If you want to receive email notifications about expired/missing 2FA credentials then uncomment 42 | # email: "sender@test.com" 43 | # Uncomment this if your SMTP username is different than your sender address (for services like AWS SES) 44 | # username: "" 45 | # default to is same as email above 46 | # to: "receiver@test.com" 47 | # password: 48 | # host: "smtp.test.com" 49 | # port: 587 50 | # If your email provider doesn't handle TLS 51 | # no_tls: true 52 | # valid values are - global (default - uses .com) or china (uses .com.cn) 53 | region: global 54 | drive: 55 | destination: "drive" 56 | # Remove local files that are not present on server (i.e. files delete on server) 57 | remove_obsolete: false 58 | sync_interval: 300 59 | filters: 60 | # File filters to be included in syncing iCloud drive content 61 | folders: 62 | - "folder1" 63 | - "folder2" 64 | - "folder3" 65 | file_extensions: 66 | # File extensions to be included 67 | - "pdf" 68 | - "png" 69 | - "jpg" 70 | - "jpeg" 71 | photos: 72 | destination: "photos" 73 | # Remove local photos that are not present on server (i.e. photos delete on server) 74 | remove_obsolete: false 75 | sync_interval: 500 76 | all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multiple albums creates duplicates on filesystem 77 | use_hardlinks: false # Optional, default false. If true and all_albums is true, create hard links for duplicate photos instead of separate copies. Saves storage space. 78 | # folder_format: "%Y/%m" # optional, if set put photos in subfolders according to format. Format cheatsheet - https://strftime.org 79 | filters: 80 | # List of libraries to download. If omitted (default), photos from all libraries (own and shared) are downloaded. If included, photos only 81 | # from the listed libraries are downloaded. 82 | # libraries: 83 | # - PrimarySync # Name of the own library 84 | 85 | # if all_albums is false - albums list is used as filter-in, if all_albums is true - albums list is used as filter-out 86 | # if albums list is empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders 87 | albums: 88 | - "album 1" 89 | - "album2" 90 | file_sizes: 91 | # valid values are original, medium and/or thumb 92 | - "original" 93 | # - "medium" 94 | # - "thumb" 95 | -------------------------------------------------------------------------------- /src/photo_file_utils.py: -------------------------------------------------------------------------------- 1 | """Photo file operations module. 2 | 3 | This module contains utilities for photo file operations including 4 | downloading, hardlink creation, and file existence checking. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | import os 10 | import shutil 11 | import time 12 | 13 | from src import get_logger 14 | 15 | LOGGER = get_logger() 16 | 17 | 18 | def check_photo_exists(photo, file_size: str, local_path: str) -> bool: 19 | """Check if photo exists locally with correct size. 20 | 21 | Args: 22 | photo: Photo object from iCloudPy 23 | file_size: File size variant (original, medium, thumb, etc.) 24 | local_path: Local file path to check 25 | 26 | Returns: 27 | True if photo exists locally with correct size, False otherwise 28 | """ 29 | if not (photo and local_path and os.path.isfile(local_path)): 30 | return False 31 | 32 | local_size = os.path.getsize(local_path) 33 | remote_size = int(photo.versions[file_size]["size"]) 34 | 35 | if local_size == remote_size: 36 | LOGGER.debug(f"No changes detected. Skipping the file {local_path} ...") 37 | return True 38 | else: 39 | LOGGER.debug(f"Change detected: local_file_size is {local_size} and remote_file_size is {remote_size}.") 40 | return False 41 | 42 | 43 | def create_hardlink(source_path: str, destination_path: str) -> bool: 44 | """Create a hard link from source to destination. 45 | 46 | Args: 47 | source_path: Path to existing file to link from 48 | destination_path: Path where hardlink should be created 49 | 50 | Returns: 51 | True if hardlink was created successfully, False otherwise 52 | """ 53 | try: 54 | # Ensure destination directory exists 55 | os.makedirs(os.path.dirname(destination_path), exist_ok=True) 56 | # Create hard link 57 | os.link(source_path, destination_path) 58 | LOGGER.info(f"Created hard link: {destination_path} (linked to existing file: {source_path})") 59 | return True 60 | except (OSError, FileNotFoundError) as e: 61 | LOGGER.warning(f"Failed to create hard link {destination_path}: {e!s}") 62 | return False 63 | 64 | 65 | def download_photo_from_server(photo, file_size: str, destination_path: str) -> bool: 66 | """Download photo from iCloud server to local path. 67 | 68 | Args: 69 | photo: Photo object from iCloudPy 70 | file_size: File size variant (original, medium, thumb, etc.) 71 | destination_path: Local path where photo should be saved 72 | 73 | Returns: 74 | True if download was successful, False otherwise 75 | """ 76 | if not (photo and file_size and destination_path): 77 | return False 78 | 79 | LOGGER.info(f"Downloading {destination_path} ...") 80 | try: 81 | download = photo.download(file_size) 82 | with open(destination_path, "wb") as file_out: 83 | shutil.copyfileobj(download.raw, file_out) 84 | 85 | # Set file modification time to photo's added date 86 | local_modified_time = time.mktime(photo.added_date.timetuple()) 87 | os.utime(destination_path, (local_modified_time, local_modified_time)) 88 | 89 | except Exception as e: 90 | # Enhanced error logging with file path context 91 | # This catches all exceptions including iCloudPy errors like ObjectNotFoundException 92 | error_msg = str(e) 93 | if "ObjectNotFoundException" in error_msg or "NOT_FOUND" in error_msg: 94 | LOGGER.error(f"Photo not found in iCloud Photos - {destination_path}: {error_msg}") 95 | else: 96 | LOGGER.error(f"Failed to download {destination_path}: {error_msg}") 97 | return False 98 | 99 | return True 100 | 101 | 102 | def rename_legacy_file_if_exists(old_path: str, new_path: str) -> None: 103 | """Rename legacy file format to new format if it exists. 104 | 105 | Args: 106 | old_path: Path to legacy file format 107 | new_path: Path to new file format 108 | """ 109 | if os.path.isfile(old_path): 110 | os.rename(old_path, new_path) 111 | -------------------------------------------------------------------------------- /src/sync_drive.py: -------------------------------------------------------------------------------- 1 | """Sync drive module. 2 | 3 | This module provides the main entry point for iCloud Drive synchronization, 4 | orchestrating the sync process using specialized utility modules per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | import unicodedata 11 | from pathlib import Path 12 | from typing import Any 13 | from urllib.parse import unquote 14 | 15 | from src import config_parser, configure_icloudpy_logging, get_logger 16 | from src.drive_cleanup import remove_obsolete # noqa: F401 17 | from src.drive_file_download import download_file # noqa: F401 18 | from src.drive_file_existence import file_exists, is_package, package_exists # noqa: F401 19 | from src.drive_filtering import ignored_path, wanted_file, wanted_folder, wanted_parent_folder # noqa: F401 20 | from src.drive_folder_processing import process_folder # noqa: F401 21 | from src.drive_package_processing import process_package # noqa: F401 22 | from src.drive_parallel_download import collect_file_for_download, download_file_task, files_lock # noqa: F401 23 | from src.drive_sync_directory import sync_directory 24 | from src.drive_thread_config import get_max_threads # noqa: F401 25 | 26 | # Configure icloudpy logging immediately after import 27 | configure_icloudpy_logging() 28 | 29 | LOGGER = get_logger() 30 | 31 | 32 | def sync_drive(config: Any, drive: Any) -> set[str]: 33 | """Synchronize iCloud Drive to local filesystem. 34 | 35 | This function serves as the main entry point for drive synchronization, 36 | preparing the destination and delegating to the sync_directory orchestrator. 37 | 38 | Args: 39 | config: Configuration dictionary containing drive settings 40 | drive: iCloud drive service instance 41 | 42 | Returns: 43 | Set of all synchronized file paths 44 | """ 45 | destination_path = config_parser.prepare_drive_destination(config=config) 46 | return sync_directory( 47 | drive=drive, 48 | destination_path=destination_path, 49 | root=destination_path, 50 | items=drive.dir(), 51 | top=True, 52 | filters=config["drive"]["filters"] if "drive" in config and "filters" in config["drive"] else None, 53 | ignore=config["drive"]["ignore"] if "drive" in config and "ignore" in config["drive"] else None, 54 | remove=config_parser.get_drive_remove_obsolete(config=config), 55 | config=config, 56 | ) 57 | 58 | 59 | def process_file(item: Any, destination_path: str, filters: list[str], ignore: list[str], files: set[str]) -> bool: 60 | """Process given item as file (legacy compatibility function). 61 | 62 | This function maintains backward compatibility with existing tests. 63 | New code should use the specialized modules directly. 64 | 65 | Args: 66 | item: iCloud file item to process 67 | destination_path: Local destination directory 68 | filters: File extension filters 69 | ignore: Ignore patterns 70 | files: Set to track processed files 71 | 72 | Returns: 73 | True if file was processed successfully, False otherwise 74 | """ 75 | if not (item and destination_path and files is not None): 76 | return False 77 | # Decode URL-encoded filename from iCloud API 78 | # This handles special characters like %CC%88 (combining diacritical marks) 79 | decoded_name = unquote(item.name) 80 | local_file = os.path.join(destination_path, decoded_name) 81 | local_file = unicodedata.normalize("NFC", local_file) 82 | if not wanted_file(filters=filters, ignore=ignore, file_path=local_file): 83 | return False 84 | files.add(local_file) 85 | item_is_package = is_package(item=item) 86 | if item_is_package: 87 | if package_exists(item=item, local_package_path=local_file): 88 | for f in Path(local_file).glob("**/*"): 89 | files.add(str(f)) 90 | return False 91 | elif file_exists(item=item, local_file=local_file): 92 | return False 93 | local_file = download_file(item=item, local_file=local_file) 94 | if local_file and item_is_package: 95 | for f in Path(local_file).glob("**/*"): 96 | f = str(f) 97 | f_normalized = unicodedata.normalize("NFD", f) 98 | if os.path.exists(f): 99 | os.rename(f, f_normalized) 100 | files.add(f_normalized) 101 | return bool(local_file) 102 | -------------------------------------------------------------------------------- /src/drive_filtering.py: -------------------------------------------------------------------------------- 1 | """Drive filtering utilities. 2 | 3 | This module provides filtering functionality for iCloud Drive sync operations, 4 | separating file and folder filtering logic from the main sync logic per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | import re 11 | from pathlib import Path, PurePath 12 | 13 | from src import configure_icloudpy_logging, get_logger 14 | 15 | # Configure icloudpy logging immediately after import 16 | configure_icloudpy_logging() 17 | 18 | LOGGER = get_logger() 19 | 20 | 21 | def wanted_file(filters: list[str] | None, ignore: list[str] | None, file_path: str) -> bool: 22 | """Check if a file should be synced based on filters and ignore patterns. 23 | 24 | Args: 25 | filters: List of file extension patterns to include (None means include all) 26 | ignore: List of ignore patterns to exclude 27 | file_path: Path to the file to check 28 | 29 | Returns: 30 | True if file should be synced, False otherwise 31 | """ 32 | if not file_path: 33 | return False 34 | 35 | if ignore and _is_ignored_path(ignore, file_path): 36 | LOGGER.debug(f"Skipping the unwanted file {file_path}") 37 | return False 38 | 39 | if not filters or len(filters) == 0: 40 | return True 41 | 42 | for file_extension in filters: 43 | if re.search(f"{file_extension}$", file_path, re.IGNORECASE): 44 | return True 45 | 46 | LOGGER.debug(f"Skipping the unwanted file {file_path}") 47 | return False 48 | 49 | 50 | def wanted_folder( 51 | filters: list[str] | None, 52 | ignore: list[str] | None, 53 | root: str, 54 | folder_path: str, 55 | ) -> bool: 56 | """Check if a folder should be synced based on filters and ignore patterns. 57 | 58 | Args: 59 | filters: List of folder paths to include (None means include all) 60 | ignore: List of ignore patterns to exclude 61 | root: Root directory path for relative path calculations 62 | folder_path: Path to the folder to check 63 | 64 | Returns: 65 | True if folder should be synced, False otherwise 66 | """ 67 | if ignore and _is_ignored_path(ignore, folder_path): 68 | return False 69 | 70 | if not filters or not folder_path or not root or len(filters) == 0: 71 | # Nothing to filter, return True 72 | return True 73 | 74 | # Something to filter 75 | folder_path = Path(folder_path) 76 | for folder in filters: 77 | child_path = Path(os.path.join(os.path.abspath(root), str(folder).removeprefix("/").removesuffix("/"))) 78 | if folder_path in child_path.parents or child_path in folder_path.parents or folder_path == child_path: 79 | return True 80 | return False 81 | 82 | 83 | def wanted_parent_folder( 84 | filters: list[str] | None, 85 | ignore: list[str] | None, 86 | root: str, 87 | folder_path: str, 88 | ) -> bool: 89 | """Check if a parent folder should be processed based on filters. 90 | 91 | Args: 92 | filters: List of folder paths to include (None means include all) 93 | ignore: List of ignore patterns to exclude 94 | root: Root directory path for relative path calculations 95 | folder_path: Path to the parent folder to check 96 | 97 | Returns: 98 | True if parent folder should be processed, False otherwise 99 | """ 100 | if not filters or not folder_path or not root or len(filters) == 0: 101 | return True 102 | 103 | folder_path = Path(folder_path) 104 | for folder in filters: 105 | child_path = Path(os.path.join(os.path.abspath(root), folder.removeprefix("/").removesuffix("/"))) 106 | if child_path in folder_path.parents or folder_path == child_path: 107 | return True 108 | return False 109 | 110 | 111 | def _is_ignored_path(ignore_list: list[str], path: str) -> bool: 112 | """Check if a path matches any ignore pattern. 113 | 114 | Args: 115 | ignore_list: List of ignore patterns 116 | path: Path to check against patterns 117 | 118 | Returns: 119 | True if path should be ignored, False otherwise 120 | """ 121 | for ignore in ignore_list: 122 | if PurePath(path).match(ignore + "*" if ignore.endswith("/") else ignore): 123 | return True 124 | return False 125 | 126 | 127 | # Legacy alias for backward compatibility 128 | ignored_path = _is_ignored_path 129 | -------------------------------------------------------------------------------- /src/drive_file_existence.py: -------------------------------------------------------------------------------- 1 | """File existence checking utilities. 2 | 3 | This module provides file and package existence checking functionality, 4 | separating existence validation logic from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | from pathlib import Path 11 | from shutil import rmtree 12 | from typing import Any 13 | 14 | from src import configure_icloudpy_logging, get_logger 15 | 16 | # Configure icloudpy logging immediately after import 17 | configure_icloudpy_logging() 18 | 19 | LOGGER = get_logger() 20 | 21 | 22 | def file_exists(item: Any, local_file: str) -> bool: 23 | """Check if a file exists locally and is up-to-date. 24 | 25 | Args: 26 | item: iCloud file item with date_modified and size attributes 27 | local_file: Path to the local file 28 | 29 | Returns: 30 | True if file exists and is up-to-date, False otherwise 31 | """ 32 | if not (item and local_file and os.path.isfile(local_file)): 33 | LOGGER.debug(f"File {local_file} does not exist locally.") 34 | return False 35 | 36 | local_file_modified_time = int(os.path.getmtime(local_file)) 37 | remote_file_modified_time = int(item.date_modified.timestamp()) 38 | local_file_size = os.path.getsize(local_file) 39 | remote_file_size = item.size 40 | 41 | if local_file_modified_time == remote_file_modified_time and ( 42 | local_file_size == remote_file_size 43 | or (local_file_size == 0 and remote_file_size is None) 44 | or (local_file_size is None and remote_file_size == 0) 45 | ): 46 | LOGGER.debug(f"No changes detected. Skipping the file {local_file} ...") 47 | return True 48 | 49 | LOGGER.debug( 50 | f"Changes detected: local_modified_time is {local_file_modified_time}, " 51 | + f"remote_modified_time is {remote_file_modified_time}, " 52 | + f"local_file_size is {local_file_size} and remote_file_size is {remote_file_size}.", 53 | ) 54 | return False 55 | 56 | 57 | def package_exists(item: Any, local_package_path: str) -> bool: 58 | """Check if a package exists locally and is up-to-date. 59 | 60 | Args: 61 | item: iCloud package item with date_modified and size attributes 62 | local_package_path: Path to the local package directory 63 | 64 | Returns: 65 | True if package exists and is up-to-date, False otherwise 66 | """ 67 | if not (item and local_package_path and os.path.isdir(local_package_path)): 68 | LOGGER.debug(f"Package {local_package_path} does not exist locally.") 69 | return False 70 | 71 | local_package_modified_time = int(os.path.getmtime(local_package_path)) 72 | remote_package_modified_time = int(item.date_modified.timestamp()) 73 | local_package_size = sum(f.stat().st_size for f in Path(local_package_path).glob("**/*") if f.is_file()) 74 | remote_package_size = item.size 75 | 76 | if local_package_modified_time == remote_package_modified_time and local_package_size == remote_package_size: 77 | LOGGER.debug(f"No changes detected. Skipping the package {local_package_path} ...") 78 | return True 79 | 80 | LOGGER.info( 81 | f"Changes detected: local_modified_time is {local_package_modified_time}, " 82 | + f"remote_modified_time is {remote_package_modified_time}, " 83 | + f"local_package_size is {local_package_size} and remote_package_size is {remote_package_size}.", 84 | ) 85 | rmtree(local_package_path) 86 | return False 87 | 88 | 89 | def is_package(item: Any) -> bool: 90 | """Determine if an iCloud item is a package that needs special handling. 91 | 92 | Args: 93 | item: iCloud item to check 94 | 95 | Returns: 96 | True if item is a package, False otherwise 97 | """ 98 | file_is_a_package = False 99 | try: 100 | with item.open(stream=True) as response: 101 | file_is_a_package = response.url and "/packageDownload?" in response.url 102 | except Exception as e: 103 | # Enhanced error logging with file context 104 | # This catches all exceptions including iCloudPy errors like ObjectNotFoundException 105 | error_msg = str(e) 106 | item_name = getattr(item, "name", "Unknown file") 107 | if "ObjectNotFoundException" in error_msg or "NOT_FOUND" in error_msg: 108 | LOGGER.error(f"File not found in iCloud Drive while checking package type - {item_name}: {error_msg}") 109 | else: 110 | LOGGER.error(f"Failed to check package type for {item_name}: {error_msg}") 111 | # Return False if we can't determine package type due to error 112 | file_is_a_package = False 113 | return file_is_a_package 114 | -------------------------------------------------------------------------------- /src/email_message.py: -------------------------------------------------------------------------------- 1 | """Email message module.""" 2 | 3 | import time 4 | import uuid 5 | from email.mime.text import MIMEText 6 | from typing import Any 7 | 8 | 9 | class EmailMessage: 10 | """ 11 | Email message class for creating and managing email messages. 12 | 13 | This class handles the creation of email messages with proper formatting 14 | and MIME structure for sending via SMTP. 15 | """ 16 | 17 | def __init__(self, **kwargs: Any) -> None: 18 | """ 19 | Initialize email message with provided parameters and defaults. 20 | 21 | Args: 22 | **kwargs: Email parameters including to, from, subject, body, etc. 23 | """ 24 | params = self._process_email_parameters(kwargs) 25 | 26 | self.to = params.get("to") 27 | self.rto = params.get("rto") 28 | self.cc = params.get("cc") 29 | self.bcc = params.get("bcc") 30 | self.sender = params.get("from") 31 | self.subject = params.get("subject", "") 32 | self.body = params.get("body") 33 | self.html = params.get("html") 34 | self.date = params.get("date", self._generate_default_date()) 35 | self.charset = params.get("charset", self._get_default_charset()) 36 | self.headers = params.get("headers", {}) 37 | 38 | self.message_id = self.make_key() 39 | 40 | def _process_email_parameters(self, kwargs: dict[str, Any]) -> dict[str, Any]: 41 | """ 42 | Process and normalize email parameters from kwargs. 43 | 44 | This function handles the transformation of keyword arguments into 45 | a normalized parameter dictionary for email message creation. 46 | 47 | Args: 48 | kwargs: Raw keyword arguments passed to constructor 49 | 50 | Returns: 51 | Dict containing processed email parameters 52 | """ 53 | params = {} 54 | for item in kwargs.items(): 55 | params[item[0]] = item[1] 56 | return params 57 | 58 | def _generate_default_date(self) -> str: 59 | """ 60 | Generate default date string in RFC 2822 format. 61 | 62 | Returns: 63 | Formatted date string for email headers 64 | """ 65 | return time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) 66 | 67 | def _get_default_charset(self) -> str: 68 | """ 69 | Get default character encoding for email messages. 70 | 71 | Returns: 72 | Default charset string 73 | """ 74 | return "us-ascii" 75 | 76 | def make_key(self) -> str: 77 | """ 78 | Generate unique message ID. 79 | 80 | Creates a unique identifier for the email message using UUID4. 81 | 82 | Returns: 83 | Unique string identifier for the message 84 | """ 85 | return str(uuid.uuid4()) 86 | 87 | def as_string(self) -> str: 88 | """ 89 | Return plaintext email content as string. 90 | 91 | This is the main public interface for getting the formatted email 92 | message ready for sending via SMTP. 93 | 94 | Returns: 95 | Complete email message as formatted string 96 | """ 97 | return self._plaintext() 98 | 99 | def _plaintext(self) -> str: 100 | """ 101 | Create plaintext email content and convert to string. 102 | 103 | Orchestrates the creation of MIME message structure and conversion 104 | to string format for email transmission. 105 | 106 | Returns: 107 | Formatted email message string 108 | """ 109 | msg = self._create_mime_message() 110 | self._set_info(msg) 111 | return msg.as_string() 112 | 113 | def _create_mime_message(self) -> MIMEText: 114 | """ 115 | Create MIME text message object. 116 | 117 | Handles the creation of the core MIME structure with proper 118 | content and encoding. 119 | 120 | Returns: 121 | MIMEText object ready for header setting 122 | """ 123 | # Handle None body by using empty string 124 | body_text = self.body if self.body is not None else "" 125 | return MIMEText(body_text, "plain", self.charset) 126 | 127 | def _set_info(self, msg: MIMEText) -> None: 128 | """ 129 | Set email header information on MIME message. 130 | 131 | Configures the essential email headers (Subject, From, To, Date) 132 | on the provided MIME message object. Handles None values gracefully. 133 | 134 | Args: 135 | msg: MIMEText object to configure with headers 136 | """ 137 | msg["Subject"] = self.subject or "" 138 | msg["From"] = self.sender or "" 139 | msg["To"] = self.to or "" 140 | msg["Date"] = self.date 141 | -------------------------------------------------------------------------------- /src/sync_stats.py: -------------------------------------------------------------------------------- 1 | """Sync statistics module for tracking sync operations and generating summaries.""" 2 | 3 | import datetime 4 | from dataclasses import dataclass, field 5 | 6 | 7 | @dataclass 8 | class DriveStats: 9 | """Statistics for drive synchronization operations. 10 | 11 | Tracks download counts, sizes, durations, and other metrics 12 | for drive sync operations. 13 | """ 14 | 15 | files_downloaded: int = 0 16 | files_skipped: int = 0 17 | files_removed: int = 0 18 | bytes_downloaded: int = 0 19 | duration_seconds: float = 0.0 20 | errors: list[str] = field(default_factory=list) 21 | 22 | def has_activity(self) -> bool: 23 | """Check if there was any sync activity. 24 | 25 | Returns: 26 | True if any files were downloaded, skipped, or removed 27 | """ 28 | return self.files_downloaded > 0 or self.files_skipped > 0 or self.files_removed > 0 29 | 30 | def has_errors(self) -> bool: 31 | """Check if there were any errors. 32 | 33 | Returns: 34 | True if errors list is not empty 35 | """ 36 | return len(self.errors) > 0 37 | 38 | 39 | @dataclass 40 | class PhotoStats: 41 | """Statistics for photo synchronization operations. 42 | 43 | Tracks download counts, hardlink usage, sizes, durations, 44 | and other metrics for photo sync operations. 45 | """ 46 | 47 | photos_downloaded: int = 0 48 | photos_hardlinked: int = 0 49 | photos_skipped: int = 0 50 | bytes_downloaded: int = 0 51 | bytes_saved_by_hardlinks: int = 0 52 | albums_synced: list[str] = field(default_factory=list) 53 | duration_seconds: float = 0.0 54 | errors: list[str] = field(default_factory=list) 55 | 56 | def has_activity(self) -> bool: 57 | """Check if there was any sync activity. 58 | 59 | Returns: 60 | True if any photos were downloaded, hardlinked, or skipped 61 | """ 62 | return self.photos_downloaded > 0 or self.photos_hardlinked > 0 or self.photos_skipped > 0 63 | 64 | def has_errors(self) -> bool: 65 | """Check if there were any errors. 66 | 67 | Returns: 68 | True if errors list is not empty 69 | """ 70 | return len(self.errors) > 0 71 | 72 | 73 | @dataclass 74 | class SyncSummary: 75 | """Overall synchronization summary combining drive and photo stats. 76 | 77 | Contains statistics for both drive and photo syncs, along with 78 | timing information for the overall sync operation. 79 | """ 80 | 81 | drive_stats: DriveStats | None = None 82 | photo_stats: PhotoStats | None = None 83 | sync_start_time: datetime.datetime = field(default_factory=datetime.datetime.now) 84 | sync_end_time: datetime.datetime | None = None 85 | 86 | def has_activity(self) -> bool: 87 | """Check if there was any sync activity overall. 88 | 89 | Returns: 90 | True if either drive or photos had activity 91 | """ 92 | drive_activity = self.drive_stats.has_activity() if self.drive_stats else False 93 | photo_activity = self.photo_stats.has_activity() if self.photo_stats else False 94 | return drive_activity or photo_activity 95 | 96 | def has_errors(self) -> bool: 97 | """Check if there were any errors in the sync. 98 | 99 | Returns: 100 | True if either drive or photos had errors 101 | """ 102 | drive_errors = self.drive_stats.has_errors() if self.drive_stats else False 103 | photo_errors = self.photo_stats.has_errors() if self.photo_stats else False 104 | return drive_errors or photo_errors 105 | 106 | def total_duration_seconds(self) -> float: 107 | """Calculate total sync duration. 108 | 109 | Returns: 110 | Total duration in seconds 111 | """ 112 | if self.sync_end_time: 113 | return (self.sync_end_time - self.sync_start_time).total_seconds() 114 | return 0.0 115 | 116 | 117 | def format_bytes(bytes_count: int) -> str: 118 | """Format byte count as human-readable string. 119 | 120 | Args: 121 | bytes_count: Number of bytes 122 | 123 | Returns: 124 | Formatted string (e.g., "1.5 GB", "234 MB") 125 | """ 126 | if bytes_count == 0: 127 | return "0 B" 128 | 129 | units = ["B", "KB", "MB", "GB", "TB"] 130 | unit_index = 0 131 | size = float(bytes_count) 132 | 133 | while size >= 1024.0 and unit_index < len(units) - 1: 134 | size /= 1024.0 135 | unit_index += 1 136 | 137 | return f"{size:.1f} {units[unit_index]}" 138 | 139 | 140 | def format_duration(seconds: float) -> str: 141 | """Format duration as human-readable string. 142 | 143 | Args: 144 | seconds: Duration in seconds 145 | 146 | Returns: 147 | Formatted string (e.g., "4m 32s", "1h 15m") 148 | """ 149 | if seconds < 60: 150 | return f"{int(seconds)}s" 151 | 152 | minutes = int(seconds // 60) 153 | remaining_seconds = int(seconds % 60) 154 | 155 | if minutes < 60: 156 | return f"{minutes}m {remaining_seconds}s" 157 | 158 | hours = minutes // 60 159 | remaining_minutes = minutes % 60 160 | return f"{hours}h {remaining_minutes}m" 161 | -------------------------------------------------------------------------------- /src/photo_path_utils.py: -------------------------------------------------------------------------------- 1 | """Photo path utils 2 | Extract filename and extension from photo. 3 | 4 | Args: 5 | photo: Photo object from iCloudPy 6 | file_size: File size variant (original, medium, thumb, etc.) 7 | 8 | Returns: 9 | Tuple of (name, extension) where name is filename without extension 10 | and extension is the file extension. 11 | 12 | This module contains utilities for generating photo file paths and managing 13 | file naming conventions for photo synchronization. 14 | """ 15 | 16 | ___author___ = "Mandar Patil " 17 | 18 | import base64 19 | import os 20 | import unicodedata 21 | from urllib.parse import unquote 22 | 23 | from src import get_logger 24 | 25 | LOGGER = get_logger() 26 | 27 | 28 | def get_photo_name_and_extension(photo, file_size: str) -> tuple[str, str]: 29 | """Extract filename and extension from photo. 30 | 31 | Args: 32 | photo: Photo object from iCloudPy 33 | file_size: File size variant (original, medium, thumb, etc.) 34 | 35 | Returns: 36 | Tuple of (name, extension) where name is filename without extension 37 | and extension is the file extension 38 | """ 39 | # Decode URL-encoded filename from iCloud API 40 | # This handles special characters like %CC%88 (combining diacritical marks) 41 | filename = unquote(photo.filename) 42 | name, extension = filename.rsplit(".", 1) if "." in filename else [filename, ""] 43 | 44 | # Handle original_alt file type mapping 45 | if file_size == "original_alt" and file_size in photo.versions: 46 | filetype = photo.versions[file_size]["type"] 47 | if filetype in _get_original_alt_filetype_mapping(): 48 | extension = _get_original_alt_filetype_mapping()[filetype] 49 | else: 50 | LOGGER.warning(f"Unknown filetype {filetype} for original_alt version of {filename}") 51 | 52 | return name, extension 53 | 54 | 55 | def generate_photo_filename_with_metadata(photo, file_size: str) -> str: 56 | """Generate filename with file size and photo ID metadata. 57 | 58 | Args: 59 | photo: Photo object from iCloudPy 60 | file_size: File size variant (original, medium, thumb, etc.) 61 | 62 | Returns: 63 | Filename string with format: name__filesize__base64id.extension 64 | """ 65 | name, extension = get_photo_name_and_extension(photo, file_size) 66 | photo_id_encoded = base64.urlsafe_b64encode(photo.id.encode()).decode() 67 | 68 | if extension == "": 69 | return f"{'__'.join([name, file_size, photo_id_encoded])}" 70 | else: 71 | return f"{'__'.join([name, file_size, photo_id_encoded])}.{extension}" 72 | 73 | 74 | def create_folder_path_if_needed(destination_path: str, folder_format: str | None, photo) -> str: 75 | """Create folder path based on folder format and photo creation date. 76 | 77 | Args: 78 | destination_path: Base destination path 79 | folder_format: strftime format string for folder creation (e.g., "%Y/%m") 80 | photo: Photo object with created date 81 | 82 | Returns: 83 | Full destination path including created folder if folder_format is specified 84 | """ 85 | if folder_format is None: 86 | return destination_path 87 | 88 | folder = photo.created.strftime(folder_format) 89 | full_destination = os.path.join(destination_path, folder) 90 | os.makedirs(full_destination, exist_ok=True) 91 | return full_destination 92 | 93 | 94 | def normalize_file_path(file_path: str) -> str: 95 | """Normalize file path using Unicode NFC normalization. 96 | 97 | Args: 98 | file_path: File path to normalize 99 | 100 | Returns: 101 | Normalized file path 102 | """ 103 | return unicodedata.normalize("NFC", file_path) 104 | 105 | 106 | def rename_legacy_file_if_exists(old_path: str, new_path: str) -> None: 107 | """Rename legacy file format to new format if it exists. 108 | 109 | Args: 110 | old_path: Path to legacy file format 111 | new_path: Path to new file format 112 | """ 113 | import os 114 | 115 | if os.path.isfile(old_path): 116 | os.rename(old_path, new_path) 117 | 118 | 119 | def _get_original_alt_filetype_mapping() -> dict: 120 | """Get mapping of original_alt file types to extensions. 121 | 122 | Returns: 123 | Dictionary mapping file types to extensions 124 | """ 125 | return { 126 | "public.png": "png", 127 | "public.jpeg": "jpeg", 128 | "public.heic": "heic", 129 | "public.image": "HEIC", 130 | "com.sony.arw-raw-image": "arw", 131 | "org.webmproject.webp": "webp", 132 | "com.compuserve.gif": "gif", 133 | "com.adobe.raw-image": "dng", 134 | "public.tiff": "tiff", 135 | "public.jpeg-2000": "jp2", 136 | "com.truevision.tga-image": "tga", 137 | "com.sgi.sgi-image": "sgi", 138 | "com.adobe.photoshop-image": "psd", 139 | "public.pbm": "pbm", 140 | "public.heif": "heif", 141 | "com.microsoft.bmp": "bmp", 142 | "com.fuji.raw-image": "raf", 143 | "com.canon.cr2-raw-image": "cr2", 144 | "com.panasonic.rw2-raw-image": "rw2", 145 | "com.nikon.nrw-raw-image": "nrw", 146 | "com.pentax.raw-image": "pef", 147 | "com.nikon.raw-image": "nef", 148 | "com.olympus.raw-image": "orf", 149 | "com.adobe.pdf": "pdf", 150 | "com.canon.cr3-raw-image": "cr3", 151 | "com.olympus.or-raw-image": "orf", 152 | "public.mpo-image": "mpo", 153 | "com.dji.mimo.pano.jpeg": "jpg", 154 | "public.avif": "avif", 155 | "com.canon.crw-raw-image": "crw", 156 | } 157 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | - Demonstrating empathy and kindness toward other people 21 | - Being respectful of differing opinions, viewpoints, and experiences 22 | - Giving and gracefully accepting constructive feedback 23 | - Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | - Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | - The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | - Trolling, insulting or derogatory comments, and personal or political attacks 33 | - Public or private harassment 34 | - Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | - Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | mandarons@pm.me. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /src/drive_parallel_download.py: -------------------------------------------------------------------------------- 1 | """Parallel download utilities. 2 | 3 | This module provides parallel download coordination for iCloud Drive sync, 4 | separating parallel execution logic from sync operations per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import os 10 | import unicodedata 11 | from concurrent.futures import ThreadPoolExecutor, as_completed 12 | from pathlib import Path 13 | from threading import Lock 14 | from typing import Any 15 | from urllib.parse import unquote 16 | 17 | from src import configure_icloudpy_logging, get_logger 18 | from src.drive_file_download import download_file 19 | from src.drive_file_existence import file_exists, is_package, package_exists 20 | from src.drive_filtering import wanted_file 21 | 22 | # Configure icloudpy logging immediately after import 23 | configure_icloudpy_logging() 24 | 25 | LOGGER = get_logger() 26 | 27 | # Thread-safe lock for file set operations 28 | files_lock = Lock() 29 | 30 | 31 | def collect_file_for_download( 32 | item: Any, 33 | destination_path: str, 34 | filters: list[str] | None, 35 | ignore: list[str] | None, 36 | files: set[str], 37 | ) -> dict[str, Any] | None: 38 | """Collect file information for parallel download without immediately downloading. 39 | 40 | Args: 41 | item: iCloud file item 42 | destination_path: Local destination directory 43 | filters: File extension filters 44 | ignore: Ignore patterns 45 | files: Set to track processed files (thread-safe updates) 46 | 47 | Returns: 48 | Download task info dict, or None if file should be skipped 49 | """ 50 | if not (item and destination_path and files is not None): 51 | return None 52 | 53 | # Decode URL-encoded filename from iCloud API 54 | # This handles special characters like %CC%88 (combining diacritical marks) 55 | decoded_name = unquote(item.name) 56 | local_file = os.path.join(destination_path, decoded_name) 57 | local_file = unicodedata.normalize("NFC", local_file) 58 | 59 | if not wanted_file(filters=filters, ignore=ignore, file_path=local_file): 60 | return None 61 | 62 | # Thread-safe file set update 63 | with files_lock: 64 | files.add(local_file) 65 | 66 | item_is_package = is_package(item=item) 67 | if item_is_package: 68 | if package_exists(item=item, local_package_path=local_file): 69 | with files_lock: 70 | for f in Path(local_file).glob("**/*"): 71 | files.add(str(f)) 72 | return None 73 | elif file_exists(item=item, local_file=local_file): 74 | return None 75 | 76 | # Return download task info 77 | return { 78 | "item": item, 79 | "local_file": local_file, 80 | "is_package": item_is_package, 81 | "files": files, 82 | } 83 | 84 | 85 | def download_file_task(download_info: dict[str, Any]) -> bool: 86 | """Download a single file as part of parallel execution. 87 | 88 | Args: 89 | download_info: Dictionary containing download task information 90 | 91 | Returns: 92 | True if download succeeded, False otherwise 93 | """ 94 | item = download_info["item"] 95 | local_file = download_info["local_file"] 96 | is_package = download_info["is_package"] 97 | files = download_info["files"] 98 | 99 | LOGGER.debug(f"[Thread] Starting download of {local_file}") 100 | 101 | try: 102 | downloaded_file = download_file(item=item, local_file=local_file) 103 | if not downloaded_file: 104 | return False 105 | 106 | if is_package: 107 | with files_lock: 108 | for f in Path(downloaded_file).glob("**/*"): 109 | f = str(f) 110 | f_normalized = unicodedata.normalize("NFD", f) 111 | if os.path.exists(f): 112 | os.rename(f, f_normalized) 113 | files.add(f_normalized) 114 | 115 | LOGGER.debug(f"[Thread] Completed download of {local_file}") 116 | return True 117 | except Exception as e: 118 | LOGGER.error(f"[Thread] Failed to download {local_file}: {e!s}") 119 | return False 120 | 121 | 122 | def execute_parallel_downloads(download_tasks: list[dict[str, Any]], max_threads: int) -> tuple[int, int]: 123 | """Execute multiple file downloads in parallel. 124 | 125 | Args: 126 | download_tasks: List of download task dictionaries 127 | max_threads: Maximum number of concurrent threads 128 | 129 | Returns: 130 | Tuple of (successful_downloads, failed_downloads) counts 131 | """ 132 | if not download_tasks: 133 | return 0, 0 134 | 135 | LOGGER.info(f"Starting parallel downloads with {max_threads} threads for {len(download_tasks)} files...") 136 | 137 | successful_downloads = 0 138 | failed_downloads = 0 139 | 140 | with ThreadPoolExecutor(max_workers=max_threads) as executor: 141 | # Submit all download tasks 142 | future_to_task = {executor.submit(download_file_task, task): task for task in download_tasks} 143 | 144 | # Process completed downloads 145 | for future in as_completed(future_to_task): 146 | try: 147 | result = future.result() 148 | if result: 149 | successful_downloads += 1 150 | else: 151 | failed_downloads += 1 152 | except Exception as e: # noqa: PERF203 153 | LOGGER.error(f"Download task failed with exception: {e!s}") 154 | failed_downloads += 1 155 | 156 | LOGGER.info(f"Parallel downloads completed: {successful_downloads} successful, {failed_downloads} failed") 157 | return successful_downloads, failed_downloads 158 | -------------------------------------------------------------------------------- /tests/test_sync_stats.py: -------------------------------------------------------------------------------- 1 | """Test for sync_stats.py file.""" 2 | 3 | import datetime 4 | import unittest 5 | 6 | from src.sync_stats import ( 7 | DriveStats, 8 | PhotoStats, 9 | SyncSummary, 10 | format_bytes, 11 | format_duration, 12 | ) 13 | 14 | 15 | class TestSyncStats(unittest.TestCase): 16 | """Tests class for sync_stats.py file.""" 17 | 18 | def test_drive_stats_has_activity(self): 19 | """Test DriveStats.has_activity() method.""" 20 | # No activity 21 | stats = DriveStats() 22 | self.assertFalse(stats.has_activity()) 23 | 24 | # Has downloaded files 25 | stats = DriveStats(files_downloaded=5) 26 | self.assertTrue(stats.has_activity()) 27 | 28 | # Has skipped files 29 | stats = DriveStats(files_skipped=10) 30 | self.assertTrue(stats.has_activity()) 31 | 32 | # Has removed files 33 | stats = DriveStats(files_removed=2) 34 | self.assertTrue(stats.has_activity()) 35 | 36 | def test_drive_stats_has_errors(self): 37 | """Test DriveStats.has_errors() method.""" 38 | # No errors 39 | stats = DriveStats() 40 | self.assertFalse(stats.has_errors()) 41 | 42 | # Has errors 43 | stats = DriveStats(errors=["error1", "error2"]) 44 | self.assertTrue(stats.has_errors()) 45 | 46 | def test_photo_stats_has_activity(self): 47 | """Test PhotoStats.has_activity() method.""" 48 | # No activity 49 | stats = PhotoStats() 50 | self.assertFalse(stats.has_activity()) 51 | 52 | # Has downloaded photos 53 | stats = PhotoStats(photos_downloaded=5) 54 | self.assertTrue(stats.has_activity()) 55 | 56 | # Has hardlinked photos 57 | stats = PhotoStats(photos_hardlinked=10) 58 | self.assertTrue(stats.has_activity()) 59 | 60 | # Has skipped photos 61 | stats = PhotoStats(photos_skipped=2) 62 | self.assertTrue(stats.has_activity()) 63 | 64 | def test_photo_stats_has_errors(self): 65 | """Test PhotoStats.has_errors() method.""" 66 | # No errors 67 | stats = PhotoStats() 68 | self.assertFalse(stats.has_errors()) 69 | 70 | # Has errors 71 | stats = PhotoStats(errors=["error1", "error2"]) 72 | self.assertTrue(stats.has_errors()) 73 | 74 | def test_sync_summary_has_activity(self): 75 | """Test SyncSummary.has_activity() method.""" 76 | # No activity 77 | summary = SyncSummary() 78 | self.assertFalse(summary.has_activity()) 79 | 80 | # Drive has activity 81 | summary = SyncSummary(drive_stats=DriveStats(files_downloaded=5)) 82 | self.assertTrue(summary.has_activity()) 83 | 84 | # Photos has activity 85 | summary = SyncSummary(photo_stats=PhotoStats(photos_downloaded=3)) 86 | self.assertTrue(summary.has_activity()) 87 | 88 | # Both have activity 89 | summary = SyncSummary( 90 | drive_stats=DriveStats(files_downloaded=5), 91 | photo_stats=PhotoStats(photos_downloaded=3), 92 | ) 93 | self.assertTrue(summary.has_activity()) 94 | 95 | def test_sync_summary_has_errors(self): 96 | """Test SyncSummary.has_errors() method.""" 97 | # No errors 98 | summary = SyncSummary() 99 | self.assertFalse(summary.has_errors()) 100 | 101 | # Drive has errors 102 | summary = SyncSummary(drive_stats=DriveStats(errors=["error"])) 103 | self.assertTrue(summary.has_errors()) 104 | 105 | # Photos has errors 106 | summary = SyncSummary(photo_stats=PhotoStats(errors=["error"])) 107 | self.assertTrue(summary.has_errors()) 108 | 109 | # Both have errors 110 | summary = SyncSummary( 111 | drive_stats=DriveStats(errors=["error1"]), 112 | photo_stats=PhotoStats(errors=["error2"]), 113 | ) 114 | self.assertTrue(summary.has_errors()) 115 | 116 | def test_sync_summary_total_duration(self): 117 | """Test SyncSummary.total_duration_seconds() method.""" 118 | # No end time 119 | summary = SyncSummary() 120 | self.assertEqual(summary.total_duration_seconds(), 0.0) 121 | 122 | # With end time 123 | start = datetime.datetime(2023, 1, 1, 10, 0, 0) 124 | end = datetime.datetime(2023, 1, 1, 10, 5, 30) 125 | summary = SyncSummary(sync_start_time=start, sync_end_time=end) 126 | self.assertEqual(summary.total_duration_seconds(), 330.0) # 5.5 minutes 127 | 128 | def test_format_bytes(self): 129 | """Test format_bytes() function.""" 130 | self.assertEqual(format_bytes(0), "0 B") 131 | self.assertEqual(format_bytes(500), "500.0 B") 132 | self.assertEqual(format_bytes(1024), "1.0 KB") 133 | self.assertEqual(format_bytes(1536), "1.5 KB") 134 | self.assertEqual(format_bytes(1048576), "1.0 MB") 135 | self.assertEqual(format_bytes(1572864), "1.5 MB") 136 | self.assertEqual(format_bytes(1073741824), "1.0 GB") 137 | self.assertEqual(format_bytes(2415919104), "2.2 GB") 138 | 139 | def test_format_duration(self): 140 | """Test format_duration() function.""" 141 | self.assertEqual(format_duration(0), "0s") 142 | self.assertEqual(format_duration(30), "30s") 143 | self.assertEqual(format_duration(59), "59s") 144 | self.assertEqual(format_duration(60), "1m 0s") 145 | self.assertEqual(format_duration(90), "1m 30s") 146 | self.assertEqual(format_duration(272), "4m 32s") 147 | self.assertEqual(format_duration(3600), "1h 0m") 148 | self.assertEqual(format_duration(4500), "1h 15m") 149 | self.assertEqual(format_duration(7265), "2h 1m") 150 | 151 | 152 | if __name__ == "__main__": 153 | unittest.main() 154 | -------------------------------------------------------------------------------- /src/album_sync_orchestrator.py: -------------------------------------------------------------------------------- 1 | """Album synchronization orchestration module. 2 | 3 | This module contains the main album sync orchestration logic 4 | that coordinates photo filtering, download collection, and parallel execution. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | import os 10 | 11 | from src import get_logger 12 | from src.hardlink_registry import HardlinkRegistry 13 | from src.photo_download_manager import ( 14 | collect_download_task, 15 | execute_parallel_downloads, 16 | ) 17 | from src.photo_filter_utils import is_photo_wanted 18 | from src.photo_path_utils import normalize_file_path 19 | 20 | LOGGER = get_logger() 21 | 22 | 23 | def sync_album_photos( 24 | album, 25 | destination_path: str, 26 | file_sizes: list[str], 27 | extensions: list[str] | None = None, 28 | files: set[str] | None = None, 29 | folder_format: str | None = None, 30 | hardlink_registry: HardlinkRegistry | None = None, 31 | config=None, 32 | ) -> bool | None: 33 | """Sync photos from given album. 34 | 35 | This function orchestrates the synchronization of a single album by: 36 | 1. Creating the destination directory 37 | 2. Collecting download tasks for wanted photos 38 | 3. Executing downloads in parallel 39 | 4. Recursively syncing subalbums 40 | 41 | Args: 42 | album: Album object from iCloudPy 43 | destination_path: Path where photos should be saved 44 | file_sizes: List of file size variants to download 45 | extensions: List of allowed file extensions (None = all allowed) 46 | files: Set to track downloaded files 47 | folder_format: strftime format string for folder organization 48 | hardlink_registry: Registry for tracking downloaded files for hardlinks 49 | config: Configuration dictionary 50 | 51 | Returns: 52 | True on success, None on invalid input 53 | """ 54 | if album is None or destination_path is None or file_sizes is None: 55 | return None 56 | 57 | # Create destination directory with normalized path 58 | normalized_destination = normalize_file_path(destination_path) 59 | os.makedirs(normalized_destination, exist_ok=True) 60 | LOGGER.info(f"Syncing {album.title}") 61 | 62 | # Collect download tasks for photos 63 | download_tasks = _collect_album_download_tasks( 64 | album, 65 | normalized_destination, 66 | file_sizes, 67 | extensions, 68 | files, 69 | folder_format, 70 | hardlink_registry, 71 | ) 72 | 73 | # Execute downloads in parallel if there are tasks 74 | if download_tasks: 75 | execute_parallel_downloads(download_tasks, config) 76 | 77 | # Recursively sync subalbums 78 | _sync_subalbums( 79 | album, 80 | normalized_destination, 81 | file_sizes, 82 | extensions, 83 | files, 84 | folder_format, 85 | hardlink_registry, 86 | config, 87 | ) 88 | 89 | return True 90 | 91 | 92 | def _collect_album_download_tasks( 93 | album, 94 | destination_path: str, 95 | file_sizes: list[str], 96 | extensions: list[str] | None, 97 | files: set[str] | None, 98 | folder_format: str | None, 99 | hardlink_registry: HardlinkRegistry | None, 100 | ) -> list: 101 | """Collect download tasks for all photos in an album. 102 | 103 | Args: 104 | album: Album object from iCloudPy 105 | destination_path: Path where photos should be saved 106 | file_sizes: List of file size variants to download 107 | extensions: List of allowed file extensions 108 | files: Set to track downloaded files 109 | folder_format: strftime format string for folder organization 110 | hardlink_registry: Registry for tracking downloaded files 111 | 112 | Returns: 113 | List of download tasks to execute 114 | """ 115 | download_tasks = [] 116 | 117 | for photo in album: 118 | if is_photo_wanted(photo, extensions): 119 | for file_size in file_sizes: 120 | download_info = collect_download_task( 121 | photo, 122 | file_size, 123 | destination_path, 124 | files, 125 | folder_format, 126 | hardlink_registry, 127 | ) 128 | if download_info: 129 | download_tasks.append(download_info) 130 | else: 131 | LOGGER.debug(f"Skipping the unwanted photo {photo.filename}.") 132 | 133 | return download_tasks 134 | 135 | 136 | def _sync_subalbums( 137 | album, 138 | destination_path: str, 139 | file_sizes: list[str], 140 | extensions: list[str] | None, 141 | files: set[str] | None, 142 | folder_format: str | None, 143 | hardlink_registry: HardlinkRegistry | None, 144 | config, 145 | ) -> None: 146 | """Recursively sync all subalbums. 147 | 148 | Args: 149 | album: Album object from iCloudPy 150 | destination_path: Base path where subalbums should be created 151 | file_sizes: List of file size variants to download 152 | extensions: List of allowed file extensions 153 | files: Set to track downloaded files 154 | folder_format: strftime format string for folder organization 155 | hardlink_registry: Registry for tracking downloaded files 156 | config: Configuration dictionary 157 | """ 158 | for subalbum in album.subalbums: 159 | sync_album_photos( 160 | album.subalbums[subalbum], 161 | os.path.join(destination_path, subalbum), 162 | file_sizes, 163 | extensions, 164 | files, 165 | folder_format, 166 | hardlink_registry, 167 | config, 168 | ) 169 | -------------------------------------------------------------------------------- /.github/workflows/ci-pr-test.yml: -------------------------------------------------------------------------------- 1 | name: CI - Pull Request 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | paths: 7 | - src/** 8 | - tests/** 9 | - Dockerfile 10 | - pylintrc 11 | - pytest.ini 12 | - requirements-test.txt 13 | - requirements.txt 14 | - run-ci.sh 15 | workflow_dispatch: 16 | jobs: 17 | cache-requirements-install: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout source code 21 | uses: actions/checkout@v4 22 | - name: Set up Python 3.10 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: "3.10" 26 | - name: Cache pip dependencies 27 | uses: actions/cache@v4 28 | id: cache-dependencies 29 | with: 30 | path: ~/.cache/pip 31 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} 32 | restore-keys: | 33 | ${{ runner.os }}-pip- 34 | - name: Install dependencies 35 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 36 | run: | 37 | pip install -r requirements-test.txt 38 | 39 | test: 40 | needs: cache-requirements-install 41 | runs-on: ubuntu-latest 42 | steps: 43 | - name: Checkout source code 44 | uses: actions/checkout@v4 45 | - name: Set up Python 3.10 46 | uses: actions/setup-python@v5 47 | with: 48 | python-version: "3.10" 49 | - name: Restore pip cache dependencies 50 | uses: actions/cache@v4 51 | with: 52 | path: ~/.cache/pip 53 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} 54 | restore-keys: | 55 | ${{ runner.os }}-pip- 56 | - name: Install dependencies 57 | run: | 58 | pip install -r requirements-test.txt 59 | - name: Test with pytest 60 | run: | 61 | sudo mkdir /config /icloud && 62 | sudo chown -R $(id -u):$(id -g) /config /icloud && 63 | ruff check && 64 | ENV_CONFIG_FILE_PATH=./tests/data/test_config.yaml pytest 65 | # - name: Setup tmate session 66 | # uses: mxschmitt/action-tmate@v3 67 | - name: Generate Allure Report 68 | uses: simple-elf/allure-report-action@master 69 | if: always() 70 | with: 71 | allure_results: allure-results 72 | allure_history: allure-history 73 | keep_reports: 100 74 | - name: Upload tests artifacts 75 | if: ${{ failure() }} 76 | uses: actions/upload-artifact@v4 77 | with: 78 | name: tests-output 79 | path: allure-history 80 | retention-days: 1 81 | - name: Upload coverage artifacts 82 | uses: actions/upload-artifact@v4 83 | if: ${{ success() }} 84 | with: 85 | name: coverage-output 86 | path: htmlcov 87 | retention-days: 1 88 | 89 | build-and-push-docker: 90 | needs: test 91 | runs-on: ubuntu-latest 92 | permissions: 93 | contents: read 94 | packages: write 95 | pull-requests: write 96 | steps: 97 | - name: Check Out Repo 98 | uses: actions/checkout@v4 99 | 100 | - name: Prepare PR tag 101 | id: prep 102 | run: | 103 | PR_NUMBER=$(echo ${{ github.event.number }}) 104 | if [ -z "$PR_NUMBER" ]; then 105 | # Fallback for workflow_dispatch or other cases 106 | PR_NUMBER=$(echo "${{ github.ref }}" | sed 's/.*\///') 107 | fi 108 | # Use lowercase repository name for GHCR 109 | REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') 110 | TAGS="ghcr.io/${REPO_NAME}:pr-${PR_NUMBER}" 111 | echo "PR Number: $PR_NUMBER" 112 | echo "Repository: $REPO_NAME" 113 | echo "Docker Tags: $TAGS" 114 | echo "tags=${TAGS}" >> $GITHUB_OUTPUT 115 | echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT 116 | 117 | - name: Login to GitHub Container Registry 118 | uses: docker/login-action@v3 119 | with: 120 | registry: ghcr.io 121 | username: ${{ github.actor }} 122 | password: ${{ secrets.GITHUB_TOKEN }} 123 | 124 | - name: Set up Docker QEMU 125 | uses: docker/setup-qemu-action@v3 126 | 127 | - name: Set up Docker Buildx 128 | id: buildx 129 | uses: docker/setup-buildx-action@v3 130 | with: 131 | version: latest 132 | driver-opts: network=host 133 | 134 | - name: Build and push 135 | id: docker_build 136 | uses: docker/build-push-action@v5 137 | with: 138 | context: ./ 139 | file: ./Dockerfile 140 | build-args: | 141 | NEW_INSTALLATION_ENDPOINT=${{ secrets.NEW_INSTALLATION_ENDPOINT }} 142 | NEW_HEARTBEAT_ENDPOINT=${{ secrets.NEW_HEARTBEAT_ENDPOINT }} 143 | APP_VERSION=pr-${{ steps.prep.outputs.pr_number }} 144 | push: true 145 | platforms: linux/amd64,linux/arm64 146 | tags: ${{ steps.prep.outputs.tags }} 147 | cache-from: type=gha 148 | cache-to: type=gha,mode=max 149 | provenance: false 150 | sbom: false 151 | 152 | - name: Image digest 153 | run: echo ${{ steps.docker_build.outputs.digest }} 154 | 155 | - name: Comment PR with Docker image info 156 | uses: actions/github-script@v7 157 | if: github.event_name == 'pull_request' 158 | with: 159 | script: | 160 | github.rest.issues.createComment({ 161 | issue_number: context.issue.number, 162 | owner: context.repo.owner, 163 | repo: context.repo.repo, 164 | body: `🐳 **Docker Image Built Successfully!** 165 | 166 | The Docker image for this PR has been built and pushed to GitHub Container Registry: 167 | 168 | **Image Tag:** \`${{ steps.prep.outputs.tags }}\` 169 | 170 | **Usage:** 171 | \`\`\`bash 172 | docker pull ${{ steps.prep.outputs.tags }} 173 | docker run --name icloud-pr-test -v \${PWD}/icloud:/icloud -v \${PWD}/config:/config -e ENV_CONFIG_FILE_PATH=/config/config.yaml ${{ steps.prep.outputs.tags }} 174 | \`\`\` 175 | 176 | **Build Info:** 177 | - Platforms: linux/arm64, linux/amd64 178 | - Image Digest: ${{ steps.docker_build.outputs.digest }} 179 | - Built from commit: ${{ github.sha }} 180 | ` 181 | }) 182 | -------------------------------------------------------------------------------- /src/drive_sync_directory.py: -------------------------------------------------------------------------------- 1 | """Drive sync directory orchestration. 2 | 3 | This module provides the main sync directory coordination functionality, 4 | orchestrating folder processing, file collection, and parallel downloads per SRP. 5 | """ 6 | 7 | __author__ = "Mandar Patil (mandarons@pm.me)" 8 | 9 | import unicodedata 10 | from typing import Any 11 | 12 | from src import configure_icloudpy_logging, get_logger, sync_drive 13 | from src.drive_cleanup import remove_obsolete 14 | from src.drive_filtering import wanted_parent_folder 15 | from src.drive_folder_processing import process_folder 16 | from src.drive_parallel_download import collect_file_for_download, execute_parallel_downloads 17 | 18 | # Configure icloudpy logging immediately after import 19 | configure_icloudpy_logging() 20 | 21 | LOGGER = get_logger() 22 | 23 | 24 | def sync_directory( 25 | drive: Any, 26 | destination_path: str, 27 | items: Any, 28 | root: str, 29 | top: bool = True, 30 | filters: dict[str, list[str]] | None = None, 31 | ignore: list[str] | None = None, 32 | remove: bool = False, 33 | config: Any | None = None, 34 | ) -> set[str]: 35 | """Synchronize a directory from iCloud Drive to local filesystem. 36 | 37 | This function orchestrates the entire sync process by: 38 | 1. Processing folders and recursively syncing subdirectories 39 | 2. Collecting files for parallel download 40 | 3. Executing parallel downloads 41 | 4. Cleaning up obsolete files if requested 42 | 43 | Args: 44 | drive: iCloud drive service instance 45 | destination_path: Local destination directory 46 | items: iCloud items to process 47 | root: Root directory for relative path calculations 48 | top: Whether this is the top-level sync call 49 | filters: Dictionary of filters (folders, file_extensions) 50 | ignore: List of ignore patterns 51 | remove: Whether to remove obsolete local files 52 | config: Configuration object 53 | 54 | Returns: 55 | Set of all processed file paths 56 | """ 57 | files = set() 58 | download_tasks = [] 59 | 60 | if not (drive and destination_path and items and root): 61 | return files 62 | 63 | # First pass: process folders and collect download tasks 64 | for i in items: 65 | item = drive[i] 66 | 67 | if item.type in ("folder", "app_library"): 68 | _process_folder_item( 69 | item, 70 | destination_path, 71 | filters, 72 | ignore, 73 | root, 74 | files, 75 | config, 76 | ) 77 | elif item.type == "file": 78 | _process_file_item( 79 | item, 80 | destination_path, 81 | filters, 82 | ignore, 83 | root, 84 | files, 85 | download_tasks, 86 | ) 87 | 88 | # Second pass: execute downloads in parallel 89 | if download_tasks: 90 | _execute_downloads(download_tasks, config) 91 | 92 | # Final cleanup if this is the top-level call 93 | if top and remove: 94 | remove_obsolete(destination_path=destination_path, files=files) 95 | 96 | return files 97 | 98 | 99 | def _process_folder_item( 100 | item: Any, 101 | destination_path: str, 102 | filters: dict[str, list[str]] | None, 103 | ignore: list[str] | None, 104 | root: str, 105 | files: set[str], 106 | config: Any | None, 107 | ) -> None: 108 | """Process a single folder item. 109 | 110 | Args: 111 | item: iCloud folder item 112 | destination_path: Local destination directory 113 | filters: Dictionary of filters 114 | ignore: List of ignore patterns 115 | root: Root directory 116 | files: Set to update with processed files 117 | config: Configuration object 118 | """ 119 | new_folder = process_folder( 120 | item=item, 121 | destination_path=destination_path, 122 | filters=filters["folders"] if filters and "folders" in filters else None, 123 | ignore=ignore, 124 | root=root, 125 | ) 126 | if not new_folder: 127 | return 128 | 129 | try: 130 | files.add(unicodedata.normalize("NFC", new_folder)) 131 | # Recursively sync subdirectory 132 | subdirectory_files = sync_directory( 133 | drive=item, 134 | destination_path=new_folder, 135 | items=item.dir(), 136 | root=root, 137 | top=False, 138 | filters=filters, 139 | ignore=ignore, 140 | config=config, 141 | ) 142 | files.update(subdirectory_files) 143 | except Exception: 144 | # Continue execution to next item, without crashing the app 145 | pass 146 | 147 | 148 | def _process_file_item( 149 | item: Any, 150 | destination_path: str, 151 | filters: dict[str, list[str]] | None, 152 | ignore: list[str] | None, 153 | root: str, 154 | files: set[str], 155 | download_tasks: list[dict[str, Any]], 156 | ) -> None: 157 | """Process a single file item. 158 | 159 | Args: 160 | item: iCloud file item 161 | destination_path: Local destination directory 162 | filters: Dictionary of filters 163 | ignore: List of ignore patterns 164 | root: Root directory 165 | files: Set to update with processed files 166 | download_tasks: List to append download tasks to 167 | """ 168 | if not wanted_parent_folder( 169 | filters=filters["folders"] if filters and "folders" in filters else None, 170 | ignore=ignore, 171 | root=root, 172 | folder_path=destination_path, 173 | ): 174 | return 175 | 176 | try: 177 | download_info = collect_file_for_download( 178 | item=item, 179 | destination_path=destination_path, 180 | filters=filters["file_extensions"] if filters and "file_extensions" in filters else None, 181 | ignore=ignore, 182 | files=files, 183 | ) 184 | if download_info: 185 | download_tasks.append(download_info) 186 | except Exception: 187 | # Continue execution to next item, without crashing the app 188 | pass 189 | 190 | 191 | def _execute_downloads(download_tasks: list[dict[str, Any]], config: Any) -> None: 192 | """Execute parallel downloads. 193 | 194 | Args: 195 | download_tasks: List of download task dictionaries 196 | config: Configuration object 197 | """ 198 | max_threads = sync_drive.get_max_threads(config) 199 | execute_parallel_downloads(download_tasks, max_threads) 200 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | """Root module.""" 2 | 3 | __author__ = "Mandar Patil (mandarons@pm.me)" 4 | 5 | import logging 6 | import os 7 | import sys 8 | import warnings 9 | 10 | from ruamel.yaml import YAML 11 | 12 | DEFAULT_ROOT_DESTINATION = "./icloud" 13 | DEFAULT_DRIVE_DESTINATION = "drive" 14 | DEFAULT_PHOTOS_DESTINATION = "photos" 15 | DEFAULT_RETRY_LOGIN_INTERVAL_SEC = 600 # 10 minutes 16 | DEFAULT_SYNC_INTERVAL_SEC = 1800 # 30 minutes 17 | DEFAULT_CONFIG_FILE_NAME = "config.yaml" 18 | ENV_ICLOUD_PASSWORD_KEY = "ENV_ICLOUD_PASSWORD" 19 | ENV_CONFIG_FILE_PATH_KEY = "ENV_CONFIG_FILE_PATH" 20 | DEFAULT_LOGGER_LEVEL = "info" 21 | DEFAULT_LOG_FILE_NAME = "icloud.log" 22 | DEFAULT_CONFIG_FILE_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), DEFAULT_CONFIG_FILE_NAME) 23 | DEFAULT_COOKIE_DIRECTORY = "/config/session_data" 24 | 25 | warnings.filterwarnings("ignore", category=DeprecationWarning) 26 | 27 | 28 | def read_config(config_path=DEFAULT_CONFIG_FILE_PATH): 29 | """Read config file.""" 30 | if not (config_path and os.path.exists(config_path)): 31 | print(f"Config file not found at {config_path}.") 32 | return None 33 | with open(file=config_path, encoding="utf-8") as config_file: 34 | config = YAML().load(config_file) 35 | config["app"]["credentials"]["username"] = ( 36 | config["app"]["credentials"]["username"].strip() if config["app"]["credentials"]["username"] is not None else "" 37 | ) 38 | return config 39 | 40 | 41 | def get_logger_config(config): 42 | """Get logger config.""" 43 | logger_config = {} 44 | if "logger" not in config["app"]: 45 | return None 46 | config_app_logger = config["app"]["logger"] 47 | logger_config["level"] = ( 48 | config_app_logger["level"].strip().lower() if "level" in config_app_logger else DEFAULT_LOGGER_LEVEL 49 | ) 50 | logger_config["filename"] = ( 51 | config_app_logger["filename"].strip().lower() if "filename" in config_app_logger else DEFAULT_LOG_FILE_NAME 52 | ) 53 | return logger_config 54 | 55 | 56 | def log_handler_exists(logger, handler_type, **kwargs): 57 | """Check for existing log handler.""" 58 | for handler in logger.handlers: 59 | if isinstance(handler, handler_type): 60 | if handler_type is logging.FileHandler: 61 | if handler.baseFilename.endswith(kwargs["filename"]): 62 | return True 63 | elif handler_type is logging.StreamHandler: 64 | if handler.stream is kwargs["stream"]: 65 | return True 66 | return False 67 | 68 | 69 | class ColorfulConsoleFormatter(logging.Formatter): 70 | """Console formatter for log messages.""" 71 | 72 | grey = "\x1b[38;21m" 73 | blue = "\x1b[38;5;39m" 74 | yellow = "\x1b[38;5;226m" 75 | red = "\x1b[38;5;196m" 76 | bold_red = "\x1b[31;1m" 77 | reset = "\x1b[0m" 78 | 79 | def __init__(self, fmt): 80 | """Construct with defaults.""" 81 | super().__init__() 82 | self.fmt = fmt 83 | self.formats = { 84 | logging.DEBUG: self.grey + self.fmt + self.reset, 85 | logging.INFO: self.blue + self.fmt + self.reset, 86 | logging.WARNING: self.yellow + self.fmt + self.reset, 87 | logging.ERROR: self.red + self.fmt + self.reset, 88 | logging.CRITICAL: self.bold_red + self.fmt + self.reset, 89 | } 90 | 91 | def format(self, record): 92 | """Format the record.""" 93 | log_fmt = self.formats.get(record.levelno) 94 | formatter = logging.Formatter(log_fmt) 95 | return formatter.format(record) 96 | 97 | 98 | def configure_icloudpy_logging(): 99 | """Configure icloudpy logging to match app logging level.""" 100 | logger_config = get_logger_config(config=read_config(config_path=os.environ.get(ENV_CONFIG_FILE_PATH_KEY, DEFAULT_CONFIG_FILE_PATH))) 101 | if logger_config: 102 | level_name = logging.getLevelName(level=logger_config["level"].upper()) 103 | 104 | # Configure icloudpy loggers to use the same level and enable propagation 105 | icloudpy_loggers = [ 106 | logging.getLogger("icloudpy"), 107 | logging.getLogger("icloudpy.base"), 108 | logging.getLogger("icloudpy.services"), 109 | logging.getLogger("icloudpy.services.photos"), 110 | ] 111 | for icloudpy_logger in icloudpy_loggers: 112 | icloudpy_logger.setLevel(level=level_name) 113 | # Enable propagation so messages go to root logger handlers 114 | icloudpy_logger.propagate = True 115 | # Remove any existing handlers to avoid duplicates 116 | icloudpy_logger.handlers.clear() 117 | 118 | 119 | def get_logger(): 120 | """Return logger.""" 121 | logger = logging.getLogger() 122 | logger_config = get_logger_config(config=read_config(config_path=os.environ.get(ENV_CONFIG_FILE_PATH_KEY, DEFAULT_CONFIG_FILE_PATH))) 123 | if logger_config: 124 | level_name = logging.getLevelName(level=logger_config["level"].upper()) 125 | logger.setLevel(level=level_name) 126 | 127 | # Create handlers once and add them to root logger 128 | file_handler = None 129 | console_handler = None 130 | 131 | if not log_handler_exists( 132 | logger=logger, 133 | handler_type=logging.FileHandler, 134 | filename=logger_config["filename"], 135 | ): 136 | file_handler = logging.FileHandler(logger_config["filename"]) 137 | file_handler.setFormatter( 138 | logging.Formatter( 139 | "%(asctime)s :: %(levelname)s :: %(name)s :: %(filename)s :: %(lineno)d :: %(message)s", 140 | ), 141 | ) 142 | logger.addHandler(file_handler) 143 | 144 | if not log_handler_exists(logger=logger, handler_type=logging.StreamHandler, stream=sys.stdout): 145 | console_handler = logging.StreamHandler(sys.stdout) 146 | console_handler.setFormatter( 147 | ColorfulConsoleFormatter( 148 | "%(asctime)s :: %(levelname)s :: %(name)s :: %(filename)s :: %(lineno)d :: %(message)s", 149 | ), 150 | ) 151 | logger.addHandler(console_handler) 152 | 153 | # Configure icloudpy loggers to use the same level and enable propagation 154 | icloudpy_loggers = [ 155 | logging.getLogger("icloudpy"), 156 | logging.getLogger("icloudpy.base"), 157 | logging.getLogger("icloudpy.services"), 158 | logging.getLogger("icloudpy.services.photos"), 159 | ] 160 | for icloudpy_logger in icloudpy_loggers: 161 | icloudpy_logger.setLevel(level=level_name) 162 | # Enable propagation so messages go to root logger handlers 163 | icloudpy_logger.propagate = True 164 | # Remove any existing handlers to avoid duplicates 165 | icloudpy_logger.handlers.clear() 166 | return logger 167 | 168 | 169 | LOGGER = get_logger() 170 | -------------------------------------------------------------------------------- /.github/workflows/ci-main-test-coverage-deploy.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: CI - Main 5 | 6 | on: 7 | push: 8 | branches: [main] 9 | paths: 10 | - src/** 11 | - tests/** 12 | - Dockerfile 13 | - pylintrc 14 | - pytest.ini 15 | - requirements-test.txt 16 | - requirements.txt 17 | - run-ci.sh 18 | workflow_dispatch: 19 | jobs: 20 | cache-requirements-install: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout source code 24 | uses: actions/checkout@v4 25 | - name: Set up Python 3.10 26 | uses: actions/setup-python@v5 27 | with: 28 | python-version: "3.10" 29 | - name: Cache pip dependencies 30 | uses: actions/cache@v4 31 | id: cache-dependencies 32 | with: 33 | path: ~/.cache/pip 34 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} 35 | restore-keys: | 36 | ${{ runner.os }}-pip- 37 | - name: Install dependencies 38 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 39 | run: | 40 | pip install -r requirements-test.txt 41 | test: 42 | needs: cache-requirements-install 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Checkout source code 46 | uses: actions/checkout@v4 47 | - name: Set up Python 3.10 48 | uses: actions/setup-python@v5 49 | with: 50 | python-version: "3.10" 51 | - name: Restore pip cache dependencies 52 | uses: actions/cache@v4 53 | with: 54 | path: ~/.cache/pip 55 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} 56 | restore-keys: | 57 | ${{ runner.os }}-pip- 58 | - name: Install dependencies 59 | run: | 60 | pip install -r requirements-test.txt 61 | - name: Test with pytest 62 | run: | 63 | sudo mkdir /config /icloud && 64 | sudo chown -R $(id -u):$(id -g) /config /icloud && 65 | ENV_CONFIG_FILE_PATH=./tests/data/test_config.yaml pytest 66 | - name: Checkout gh-pages 67 | uses: actions/checkout@v4 68 | if: always() 69 | continue-on-error: true 70 | with: 71 | ref: gh-pages 72 | path: gh-pages 73 | - name: Generate Allure Report 74 | uses: simple-elf/allure-report-action@master 75 | if: always() 76 | with: 77 | allure_results: allure-results 78 | subfolder: test-results 79 | allure_report: allure-report 80 | allure_history: allure-history 81 | keep_reports: 100 82 | - name: Zip tests artifacts 83 | run: zip test-output.zip allure-history/* -r 84 | - name: Generate badges 85 | run: | 86 | python generate_badges.py 87 | - name: Upload tests artifacts 88 | uses: actions/upload-artifact@v4 89 | with: 90 | name: test-output.zip 91 | path: test-output.zip 92 | retention-days: 1 93 | - name: Upload coverage artifacts 94 | uses: actions/upload-artifact@v4 95 | with: 96 | name: coverage-output 97 | path: htmlcov 98 | retention-days: 1 99 | overwrite: true 100 | - name: Upload badges artifacts 101 | uses: actions/upload-artifact@v4 102 | with: 103 | name: badges-output 104 | path: badges 105 | retention-days: 1 106 | publish-test-report: 107 | needs: test 108 | runs-on: ubuntu-latest 109 | steps: 110 | - name: Download test artifacts 111 | uses: actions/download-artifact@v4 112 | with: 113 | name: test-output.zip 114 | # - name: Setup tmate session 115 | # uses: mxschmitt/action-tmate@v3 116 | - name: Checkout gh-pages 117 | uses: actions/checkout@v4 118 | if: always() 119 | continue-on-error: true 120 | with: 121 | ref: gh-pages 122 | path: gh-pages 123 | - name: unzip tests artifacts 124 | run: | 125 | unzip test-output.zip && rm test-output.zip 126 | - name: Publish test report to gh-pages 127 | if: always() 128 | uses: peaceiris/actions-gh-pages@v4 129 | with: 130 | deploy_key: ${{ secrets.DEPLOY_PRIVATE_KEY }} 131 | publish_branch: gh-pages 132 | publish_dir: allure-history/test-results 133 | destination_dir: test-results 134 | publish-coverage-report: 135 | needs: publish-test-report 136 | runs-on: ubuntu-latest 137 | steps: 138 | - name: Download coverage artifacts 139 | uses: actions/download-artifact@v4 140 | with: 141 | name: coverage-output 142 | path: coverage 143 | - name: Checkout gh-pages 144 | uses: actions/checkout@v4 145 | if: always() 146 | continue-on-error: true 147 | with: 148 | ref: gh-pages 149 | path: gh-pages 150 | - name: Publish test coverage to gh-pages 151 | if: always() 152 | uses: peaceiris/actions-gh-pages@v4 153 | with: 154 | deploy_key: ${{ secrets.DEPLOY_PRIVATE_KEY }} 155 | publish_branch: gh-pages 156 | publish_dir: coverage 157 | destination_dir: test-coverage 158 | publish-badges: 159 | needs: publish-coverage-report 160 | runs-on: ubuntu-latest 161 | steps: 162 | - name: Download badges artifacts 163 | uses: actions/download-artifact@v4 164 | with: 165 | name: badges-output 166 | path: badges 167 | - name: Checkout gh-pages 168 | uses: actions/checkout@v4 169 | if: always() 170 | continue-on-error: true 171 | with: 172 | ref: gh-pages 173 | path: gh-pages 174 | - name: Publish badges to gh-pages 175 | if: always() 176 | uses: peaceiris/actions-gh-pages@v4 177 | with: 178 | deploy_key: ${{ secrets.DEPLOY_PRIVATE_KEY }} 179 | publish_branch: gh-pages 180 | publish_dir: badges 181 | destination_dir: badges 182 | deploy: 183 | needs: test 184 | runs-on: ubuntu-latest 185 | steps: 186 | - name: Check Out Repo 187 | uses: actions/checkout@v4 188 | - name: Cache Docker layers 189 | uses: actions/cache@v4 190 | with: 191 | path: /tmp/.buildx-cache 192 | key: ${{ runner.os }}-buildx-${{ github.sha }} 193 | restore-keys: | 194 | ${{ runner.os }}-buildx- 195 | - name: Login to Docker Hub 196 | uses: docker/login-action@v3 197 | with: 198 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 199 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 200 | - name: Set up Docker QEMU 201 | uses: docker/setup-qemu-action@v3 202 | - name: Set up Docker Buildx 203 | id: buildx 204 | uses: docker/setup-buildx-action@v3 205 | - name: Build and push 206 | id: docker_build 207 | uses: docker/build-push-action@v5 208 | with: 209 | context: ./ 210 | file: ./Dockerfile 211 | build-args: | 212 | NEW_INSTALLATION_ENDPOINT=${{ secrets.NEW_INSTALLATION_ENDPOINT }} 213 | NEW_HEARTBEAT_ENDPOINT=${{ secrets.NEW_HEARTBEAT_ENDPOINT }} 214 | APP_VERSION=main 215 | push: true 216 | platforms: linux/arm64, linux/amd64 217 | tags: mandarons/icloud-drive:main 218 | cache-from: type=local,src=/tmp/.buildx-cache 219 | cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max 220 | - # Temp fix 221 | # https://github.com/docker/build-push-action/issues/252 222 | # https://github.com/moby/buildkit/issues/1896 223 | name: Move cache 224 | run: | 225 | rm -rf /tmp/.buildx-cache 226 | mv /tmp/.buildx-cache-new /tmp/.buildx-cache 227 | - name: Image digest 228 | run: echo ${{ steps.docker_build.outputs.digest }} 229 | -------------------------------------------------------------------------------- /src/photo_download_manager.py: -------------------------------------------------------------------------------- 1 | """Photo download task management module. 2 | 3 | This module contains utilities for managing photo download tasks 4 | and parallel execution during photo synchronization. 5 | """ 6 | 7 | ___author___ = "Mandar Patil " 8 | 9 | import os 10 | from concurrent.futures import ThreadPoolExecutor, as_completed 11 | from threading import Lock 12 | 13 | from src import config_parser, get_logger 14 | from src.hardlink_registry import HardlinkRegistry 15 | from src.photo_file_utils import create_hardlink, download_photo_from_server 16 | from src.photo_path_utils import ( 17 | create_folder_path_if_needed, 18 | generate_photo_filename_with_metadata, 19 | normalize_file_path, 20 | rename_legacy_file_if_exists, 21 | ) 22 | 23 | LOGGER = get_logger() 24 | 25 | # Thread-safe lock for file set operations 26 | files_lock = Lock() 27 | 28 | 29 | class DownloadTaskInfo: 30 | """Information about a photo download task.""" 31 | 32 | def __init__(self, photo, file_size: str, photo_path: str, 33 | hardlink_source: str | None = None, 34 | hardlink_registry: HardlinkRegistry | None = None): 35 | """Initialize download task info. 36 | 37 | Args: 38 | photo: Photo object from iCloudPy 39 | file_size: File size variant (original, medium, thumb, etc.) 40 | photo_path: Target path for photo download 41 | hardlink_source: Path to existing file for hardlink creation 42 | hardlink_registry: Registry for tracking downloaded files 43 | """ 44 | self.photo = photo 45 | self.file_size = file_size 46 | self.photo_path = photo_path 47 | self.hardlink_source = hardlink_source 48 | self.hardlink_registry = hardlink_registry 49 | 50 | 51 | def get_max_threads_for_download(config) -> int: 52 | """Get maximum number of threads for parallel downloads. 53 | 54 | Args: 55 | config: Configuration dictionary 56 | 57 | Returns: 58 | Maximum number of threads to use for downloads 59 | """ 60 | return config_parser.get_app_max_threads(config) 61 | 62 | 63 | def generate_photo_path(photo, file_size: str, destination_path: str, 64 | folder_format: str | None) -> str: 65 | """Generate full file path for photo with legacy file renaming. 66 | 67 | This function combines path generation, folder creation, and legacy 68 | file renaming into a single operation to maintain backward compatibility. 69 | 70 | Args: 71 | photo: Photo object from iCloudPy 72 | file_size: File size variant (original, medium, thumb, etc.) 73 | destination_path: Base destination path 74 | folder_format: strftime format string for folder creation 75 | 76 | Returns: 77 | Normalized full path where photo should be saved 78 | """ 79 | # Generate filename with metadata 80 | filename_with_metadata = generate_photo_filename_with_metadata(photo, file_size) 81 | 82 | # Create folder path if needed 83 | final_destination = create_folder_path_if_needed(destination_path, folder_format, photo) 84 | 85 | # Generate paths for legacy file format handling 86 | filename = photo.filename 87 | name, extension = filename.rsplit(".", 1) if "." in filename else [filename, ""] 88 | 89 | # Legacy file paths that need to be renamed 90 | file_path = os.path.join(destination_path, filename) 91 | file_size_path = os.path.join( 92 | destination_path, 93 | f"{'__'.join([name, file_size])}" if extension == "" else f"{'__'.join([name, file_size])}.{extension}", 94 | ) 95 | 96 | # Final path with normalization 97 | final_file_path = os.path.join(final_destination, filename_with_metadata) 98 | normalized_path = normalize_file_path(final_file_path) 99 | 100 | # Rename legacy files if they exist 101 | rename_legacy_file_if_exists(file_path, normalized_path) 102 | rename_legacy_file_if_exists(file_size_path, normalized_path) 103 | 104 | # Handle existing file with different normalization 105 | if os.path.isfile(final_file_path) and final_file_path != normalized_path: 106 | rename_legacy_file_if_exists(final_file_path, normalized_path) 107 | 108 | return normalized_path 109 | 110 | 111 | def collect_download_task(photo, file_size: str, destination_path: str, 112 | files: set[str] | None, folder_format: str | None, 113 | hardlink_registry: HardlinkRegistry | None) -> DownloadTaskInfo | None: 114 | """Collect photo info for parallel download without immediately downloading. 115 | 116 | Args: 117 | photo: Photo object from iCloudPy 118 | file_size: File size variant (original, medium, thumb, etc.) 119 | destination_path: Base destination path 120 | files: Set to track downloaded files (thread-safe updates) 121 | folder_format: strftime format string for folder creation 122 | hardlink_registry: Registry for tracking downloaded files 123 | 124 | Returns: 125 | DownloadTaskInfo if photo needs to be processed, None if skipped 126 | """ 127 | # Check if file size exists on server 128 | if file_size not in photo.versions: 129 | photo_path = generate_photo_path(photo, file_size, destination_path, folder_format) 130 | LOGGER.warning(f"File size {file_size} not found on server. Skipping the photo {photo_path} ...") 131 | return None 132 | 133 | # Generate photo path 134 | photo_path = generate_photo_path(photo, file_size, destination_path, folder_format) 135 | 136 | # Thread-safe file set update 137 | if files is not None: 138 | with files_lock: 139 | files.add(photo_path) 140 | 141 | # Check if photo already exists with correct size 142 | from src.photo_file_utils import check_photo_exists 143 | if check_photo_exists(photo, file_size, photo_path): 144 | return None 145 | 146 | # Check for existing hardlink source 147 | hardlink_source = None 148 | if hardlink_registry is not None: 149 | hardlink_source = hardlink_registry.get_existing_path(photo.id, file_size) 150 | 151 | return DownloadTaskInfo( 152 | photo=photo, 153 | file_size=file_size, 154 | photo_path=photo_path, 155 | hardlink_source=hardlink_source, 156 | hardlink_registry=hardlink_registry, 157 | ) 158 | 159 | 160 | def execute_download_task(task_info: DownloadTaskInfo) -> bool: 161 | """Download a single photo or create hardlink as part of parallel execution. 162 | 163 | Args: 164 | task_info: Download task information 165 | 166 | Returns: 167 | True if task completed successfully, False otherwise 168 | """ 169 | LOGGER.debug(f"[Thread] Starting processing of {task_info.photo_path}") 170 | 171 | try: 172 | # Try hardlink first if source exists 173 | if task_info.hardlink_source: 174 | if create_hardlink(task_info.hardlink_source, task_info.photo_path): 175 | LOGGER.debug(f"[Thread] Created hardlink for {task_info.photo_path}") 176 | return True 177 | else: 178 | # Fallback to download if hard link creation fails 179 | LOGGER.warning(f"Hard link creation failed, downloading {task_info.photo_path} instead") 180 | 181 | # Download the photo 182 | result = download_photo_from_server(task_info.photo, task_info.file_size, task_info.photo_path) 183 | if result and task_info.hardlink_registry is not None: 184 | # Register for future hard links if enabled 185 | task_info.hardlink_registry.register_photo_path( 186 | task_info.photo.id, task_info.file_size, task_info.photo_path, 187 | ) 188 | LOGGER.debug(f"[Thread] Completed download of {task_info.photo_path}") 189 | 190 | return result 191 | 192 | except Exception as e: 193 | LOGGER.error(f"[Thread] Failed to process {task_info.photo_path}: {e!s}") 194 | return False 195 | 196 | 197 | def execute_parallel_downloads(download_tasks: list[DownloadTaskInfo], config) -> tuple[int, int]: 198 | """Execute download tasks in parallel using thread pool. 199 | 200 | Args: 201 | download_tasks: List of download tasks to execute 202 | config: Configuration dictionary for thread settings 203 | 204 | Returns: 205 | Tuple of (successful_downloads, failed_downloads) 206 | """ 207 | if not download_tasks: 208 | return 0, 0 209 | 210 | max_threads = get_max_threads_for_download(config) 211 | 212 | # Count hardlink tasks vs download tasks for logging 213 | hardlink_tasks = sum(1 for task in download_tasks if task.hardlink_source) 214 | download_only_tasks = len(download_tasks) - hardlink_tasks 215 | 216 | if hardlink_tasks > 0: 217 | LOGGER.info( 218 | f"Starting parallel processing with {max_threads} threads: " 219 | f"{hardlink_tasks} hard links, {download_only_tasks} downloads...", 220 | ) 221 | else: 222 | LOGGER.info( 223 | f"Starting parallel photo downloads with {max_threads} threads for {len(download_tasks)} photos...", 224 | ) 225 | 226 | successful_downloads = 0 227 | failed_downloads = 0 228 | 229 | with ThreadPoolExecutor(max_workers=max_threads) as executor: 230 | # Submit all download tasks 231 | future_to_task = {executor.submit(execute_download_task, task): task for task in download_tasks} 232 | 233 | # Process completed downloads 234 | for future in as_completed(future_to_task): 235 | try: 236 | result = future.result() 237 | if result: 238 | successful_downloads += 1 239 | else: 240 | failed_downloads += 1 241 | except Exception as e: # noqa: PERF203 242 | LOGGER.error(f"Unexpected error during photo download: {e!s}") 243 | failed_downloads += 1 244 | 245 | LOGGER.info(f"Photo processing complete: {successful_downloads} successful, {failed_downloads} failed") 246 | return successful_downloads, failed_downloads 247 | -------------------------------------------------------------------------------- /.github/copilot-instructions.md: -------------------------------------------------------------------------------- 1 | # iCloud Docker - AI Coding Agent Instructions 2 | 3 | ## Project Overview 4 | This is a containerized iCloud sync client that downloads files/photos from iCloud Drive and Photos to local filesystem. Built as a long-running Docker service using Alpine Linux with custom process management (no S6 overlay - uses `su-exec` and `docker-entrypoint.sh`). 5 | 6 | ## Architecture & Core Components 7 | 8 | ### Main Sync Loop (`src/sync.py`) - **RECENTLY REFACTORED** 9 | - **Orchestrator pattern**: Main `sync()` function delegates to 15+ focused helper functions following SRP 10 | - **State management**: `SyncState` class encapsulates countdown timers and sync flags to avoid parameter passing 11 | - **Adaptive scheduling**: `_calculate_next_sync_schedule()` determines which service syncs next based on timers 12 | - Tracks `drive_time_remaining` and `photos_time_remaining` in state object 13 | - Subtracts elapsed time from other service's timer when one service syncs 14 | - **Oneshot mode**: Set sync_interval to `-1` to run once and exit (useful for cron-style scheduling) 15 | - Exit condition: `_should_exit_oneshot_mode()` checks if ALL configured intervals are negative 16 | - **2FA handling**: `_handle_2fa_required()` manages authentication retry with configurable intervals 17 | - **Password management**: `_retrieve_password()` uses keyring storage, falls back to `ENV_ICLOUD_PASSWORD` 18 | - Stores password from env var into keyring on first run for persistence 19 | 20 | ### Configuration System - **HEAVILY REFACTORED** 21 | - **Layered architecture**: Core utilities in `config_utils.py`, logging in `config_logging.py`, main API in `config_parser.py` 22 | - **YAML-based** with deep path traversal: `traverse_config_path()` → `get_config_value()` pattern (NEVER direct dict access) 23 | - **Runtime config reloading**: Config re-read on every sync loop iteration via `_load_configuration()` 24 | - **Environment overrides**: `ENV_CONFIG_FILE_PATH` and `ENV_ICLOUD_PASSWORD` override file values 25 | - **Validation pattern**: ALWAYS use `traverse_config_path()` before `get_config_value()` to avoid KeyError 26 | - **Thread configuration**: `get_app_max_threads()` supports `"auto"` (caps at 8) or integer 1-16 27 | - **Zero duplication**: Shared patterns extracted into helper functions (e.g., `get_sync_interval()`) 28 | 29 | ### Drive Sync Modules - **SPLIT INTO 8 SPECIALIZED MODULES** 30 | - **`sync_drive.py`**: High-level orchestration and folder processing 31 | - **`drive_parallel_download.py`**: Parallel download coordination with `ThreadPoolExecutor` 32 | - **`drive_file_download.py`**: Individual file download and atomic operations 33 | - **`drive_filtering.py`**: File/folder filtering logic via glob matching 34 | - **`drive_file_existence.py`**: File existence checks and package detection 35 | - **`drive_cleanup.py`**: Obsolete file removal when `remove_obsolete` enabled 36 | - **`drive_package_processing.py`**: ZIP auto-extraction and gzip handling with `magic` library 37 | - **`drive_folder_processing.py`**: Directory traversal and recursive processing 38 | 39 | ### Photos Sync Modules - **SPLIT INTO 7 SPECIALIZED MODULES** 40 | - **`sync_photos.py`**: High-level orchestration (libraries vs albums) 41 | - **`album_sync_orchestrator.py`**: Album synchronization coordination 42 | - **`photo_download_manager.py`**: Parallel download task collection and execution 43 | - **`photo_filter_utils.py`**: Photo filtering by extensions and album preferences 44 | - **`photo_path_utils.py`**: Path normalization and folder format handling 45 | - **`photo_file_utils.py`**: File operations and metadata handling 46 | - **`hardlink_registry.py`**: `HardlinkRegistry` class for deduplication across albums 47 | - **File sizes**: `original`, `original_alt` (RAW fallback), `medium`, `thumb` 48 | - **Hardlink deduplication**: `use_hardlinks` mode with registry tracking across albums 49 | - **Date organization**: `folder_format` uses strftime patterns (e.g., `"%Y/%m"`) 50 | 51 | ### Notification System (`src/notify.py`) 52 | - **Multi-provider**: Discord, Telegram, Pushover, SMTP with rate limiting (24-hour throttle) 53 | - **2FA alerts**: Automatically notifies when authentication expires (`api.requires_2sa`) 54 | - **Rate limiting**: `last_send` parameter prevents notification spam (returns same timestamp if < 24hrs) 55 | 56 | ### Usage Tracking (`src/usage.py`) 57 | - **Opt-in telemetry**: Collects anonymized sync statistics for usage analytics (see `USAGE.md`) 58 | - **Opt-out**: Set `app.usage_tracking.enabled: false` in config to disable completely 59 | - **Data collected**: Version, sync duration/counts, error indicators (no file names/paths/credentials) 60 | - **Endpoints**: `NEW_INSTALLATION_ENDPOINT` and `NEW_HEARTBEAT_ENDPOINT` from Dockerfile build args 61 | 62 | ## Development Workflow 63 | 64 | ### Local Testing 65 | ```bash 66 | # Run full CI pipeline locally (includes ruff, pytest, allure report) 67 | source .venv/bin/activate && ./run-ci.sh 68 | # Manually: ruff check --fix && ENV_CONFIG_FILE_PATH=./tests/data/test_config.yaml pytest 69 | ``` 70 | 71 | ### Key Testing Patterns 72 | - **Mock strategy**: Tests use `ICloudPyServiceMock` in `tests/data/__init__.py` (4000+ lines) with photo fixtures in `tests/data/photos_data.py` (2800+ lines) 73 | - **Config injection**: Tests override config paths via `tests.CONFIG_PATH` and `tests.TEMP_DIR` 74 | - **100% coverage requirement**: `pytest.ini` enforces `--cov-fail-under=100` (build fails below 100%) 75 | - **Temp directory cleanup**: All tests use `setUp()/tearDown()` pattern to clean `tests.TEMP_DIR` 76 | - **Allure reporting**: CI generates test reports via `allure generate --clean` 77 | - **Test structure**: Each `src/*.py` has mirror `tests/test_*.py` (e.g., `sync.py` → `test_sync.py`) 78 | 79 | ### Docker Development 80 | - **Base image**: `python:3.10-alpine3.22` (multi-stage build with builder pattern) 81 | - **Service management**: Entry point is `/usr/local/bin/docker-entrypoint.sh` → calls `/app/init.sh` → runs `python src/main.py` 82 | - **User management**: Creates `abc` user with configurable PUID/PGID (default 911:911) 83 | - **Debug container**: `Dockerfile-debug` includes `debugpy` on port 5678 for remote debugging 84 | - **Volume ownership**: Entrypoint sets ownership on `/app`, `/config`, `/icloud`, `/home/abc` if needed 85 | 86 | ### CI/CD Pipeline 87 | - **Workflow**: `.github/workflows/ci-main-test-coverage-deploy.yml` runs on main branch 88 | - **Steps**: Cache pip → Run tests → Generate coverage badges → Deploy to GitHub Pages 89 | - **Triggers**: Changes to `src/**`, `tests/**`, `Dockerfile`, `pytest.ini`, `requirements*.txt` 90 | - **PR checks**: `.github/workflows/ci-pr-test.yml` runs ruff + pytest on PRs 91 | 92 | ## Critical Implementation Details 93 | 94 | ### Authentication Flow 95 | 1. Check `ENV_ICLOUD_PASSWORD` environment variable first 96 | 2. If set, store in keyring via `utils.store_password_in_keyring()` for persistence 97 | 3. Otherwise, retrieve from keyring via `utils.get_password_from_keyring()` 98 | 4. On 2FA requirement (`api.requires_2sa`), enter retry loop with `retry_login_interval` 99 | 5. For China region, use different endpoints in `get_api_instance()`: 100 | - Home: `https://www.icloud.com.cn` 101 | - Setup: `https://setup.icloud.com.cn/setup/ws/1` 102 | 103 | ### File Handling Patterns 104 | - **Path normalization**: All modules use `unicodedata.normalize("NFC", path)` for macOS/Windows compatibility 105 | - Also normalize to NFD when searching/comparing existing files 106 | - **Atomic operations**: Files downloaded to temp paths, then moved to final location 107 | - **Compression handling**: 108 | - `drive_package_processing.py` detects ZIP via `magic.from_file()` and auto-extracts 109 | - Handles gzip streams with `gzip.open()` for compressed downloads 110 | - **Thread safety**: Use `files_lock` when modifying shared `files` set in parallel download workers 111 | 112 | ### Refactoring Principles - **CRITICAL FOR NEW CODE** 113 | - **Single Responsibility Principle**: Every function has ONE clear purpose - modules split from monoliths 114 | - **No code duplication**: Extract common patterns into utilities (e.g., `config_utils.py`, `filesystem_utils.py`) 115 | - **Separation of concerns**: Business logic, logging, and configuration parsing are separate 116 | - **State management**: Use classes like `SyncState` and `HardlinkRegistry` instead of parameter passing 117 | - **Layered architecture**: Core utilities → Business logic → Orchestration layers 118 | - **Example refactor**: `sync.py` main loop uses 15+ focused helpers; drive/photos split into 8/7 modules respectively 119 | 120 | ### Error Handling Strategy 121 | - **Graceful degradation**: Missing config sections disable features rather than crash 122 | - Example: No `drive` section means skip drive sync entirely 123 | - **Network errors**: iCloudPy has built-in retry logic for transient failures 124 | - **Clean exits**: Oneshot mode uses `_should_exit_oneshot_mode()` to break main loop after single sync 125 | - **2FA expiry**: Sends notifications and retries login based on `retry_login_interval` (-1 = exit immediately) 126 | 127 | ### Container Integration 128 | - **User management**: Runs as `abc` user (PUID/PGID configurable via env vars) 129 | - **Volume mounts**: 130 | - `/config` → config.yaml + session_data/ 131 | - `/icloud` → synced content (drive/ and photos/) 132 | - `/home/abc/.local` → Optional keyring persistence 133 | - **Session persistence**: Authentication tokens in `/config/session_data` (iCloudPy cookie directory) 134 | - **Init sequence**: `docker-entrypoint.sh` → sets UID/GID → `su-exec abc /app/init.sh` → `python src/main.py` 135 | 136 | ## Code Conventions 137 | - **Module structure**: Each `src/*.py` has corresponding `tests/test_*.py` mirror 138 | - **Logging pattern**: Use `LOGGER = get_logger()` at module level (never instantiate inline) 139 | - **iCloudPy logging**: Call `configure_icloudpy_logging()` immediately after imports to suppress verbose logs 140 | - **Config access**: NEVER use direct dict access (`config["key"]`) - always use `config_parser.get_*()` functions 141 | - **Error messages**: Include config path context: `config_path_to_string(config_path)` for debugging 142 | - **Constants**: Define in `src/__init__.py` (e.g., `DEFAULT_COOKIE_DIRECTORY = "/config/session_data"`) 143 | - **Type annotations**: All new functions must have comprehensive type hints and docstrings 144 | 145 | ## External Dependencies 146 | - **iCloudPy** (0.7.0): Core iCloud API client (`from icloudpy import ICloudPyService`) 147 | - Runtime uses PyPI package, not the `external/icloudpy` git submodule 148 | - Submodule is included for reference/development purposes only 149 | - **ruamel.yaml** (0.18.15): YAML parsing with comment preservation 150 | - **python-magic** (0.4.27): File type detection for drive sync (requires `libmagic` at runtime) 151 | - **requests** (~2.32.3): HTTP client for notifications and usage tracking 152 | - **Container tools**: `su-exec` (setuid alternative), `shadow` (user management) 153 | 154 | ## Common Pitfalls 155 | - **Refactoring violations**: Don't create monolithic functions - break into SRP-compliant helpers 156 | - **Config validation**: Missing `traverse_config_path()` check will cause KeyError crashes 157 | - **Thread limits**: Don't exceed 16 max_threads (server protection) - auto caps at min(CPU, 8) 158 | - **Test coverage**: 100% required - new code MUST have corresponding test cases in `tests/` 159 | - **Unicode normalization**: Always normalize file paths (NFC for storage, NFD for comparison) 160 | - **State management**: Use classes for complex state instead of passing many parameters -------------------------------------------------------------------------------- /NOTIFICATION_CONFIG.md: -------------------------------------------------------------------------------- 1 | # Notification Configuration Guide 2 | 3 | iCloud-docker supports comprehensive notification capabilities to keep you informed about sync operations, authentication status, and potential issues. This guide covers all notification features and configuration options. 4 | 5 | ## Overview 6 | 7 | The notification system supports two main types of alerts: 8 | 9 | 1. **2FA Authentication Alerts** - Critical notifications when iCloud authentication expires 10 | 2. **Sync Summary Notifications** - Detailed reports after each sync cycle with statistics 11 | 12 | ## Notification Services 13 | 14 | ### Supported Services 15 | 16 | | Service | Use Case | Features | 17 | |---------|----------|----------| 18 | | **Discord** | Team/Server notifications | Rich formatting, webhooks, persistent history | 19 | | **Telegram** | Personal/Mobile alerts | Instant delivery, group support, multimedia | 20 | | **Pushover** | Dedicated mobile notifications | Priority levels, custom sounds, offline delivery | 21 | | **Email (SMTP)** | Universal compatibility | UTF-8 support, custom formatting, professional | 22 | 23 | ### Service Configuration 24 | 25 | #### Discord 26 | ```yaml 27 | app: 28 | discord: 29 | webhook_url: "https://discord.com/api/webhooks/YOUR_WEBHOOK_ID/YOUR_WEBHOOK_TOKEN" 30 | username: "icloud-sync" # Optional: Custom bot name (default: icloud-docker) 31 | ``` 32 | 33 | **Setup Steps:** 34 | 1. Go to your Discord server settings 35 | 2. Navigate to Integrations → Webhooks 36 | 3. Create a new webhook or edit existing one 37 | 4. Copy the webhook URL 38 | 5. Optionally customize the username 39 | 40 | #### Telegram 41 | ```yaml 42 | app: 43 | telegram: 44 | bot_token: "1234567890:ABCdefGHIjklMNOpqrsTUVwxyz" 45 | chat_id: "123456789" # Can be user ID or group chat ID 46 | ``` 47 | 48 | **Setup Steps:** 49 | 1. Message @BotFather on Telegram 50 | 2. Use `/newbot` command and follow instructions 51 | 3. Save the bot token provided 52 | 4. Add your bot to desired chat or use personal chat 53 | 5. Get your chat ID using @userinfobot or @RawDataBot 54 | 55 | #### Pushover 56 | ```yaml 57 | app: 58 | pushover: 59 | user_key: "your-30-char-user-key" 60 | api_token: "your-30-char-app-token" 61 | ``` 62 | 63 | **Setup Steps:** 64 | 1. Sign up at [Pushover.net](https://pushover.net) 65 | 2. Note your user key from the dashboard 66 | 3. Create a new application to get an API token 67 | 4. Install Pushover app on your mobile device 68 | 69 | #### Email (SMTP) 70 | ```yaml 71 | app: 72 | smtp: 73 | email: "icloud-sync@yourdomain.com" # Sender address 74 | to: "admin@yourdomain.com" # Recipient (optional, defaults to sender) 75 | username: "smtp-username" # Optional: If different from email 76 | password: "your-app-password" # App password or SMTP password 77 | host: "smtp.gmail.com" # SMTP server 78 | port: 587 # SMTP port (587 for TLS, 465 for SSL, 25 for plain) 79 | no_tls: false # Set to true if TLS is not supported 80 | ``` 81 | 82 | **Popular SMTP Settings:** 83 | - **Gmail**: `smtp.gmail.com:587` (requires app password) 84 | - **Outlook**: `smtp-mail.outlook.com:587` 85 | - **Yahoo**: `smtp.mail.yahoo.com:587` 86 | - **AWS SES**: `email-smtp.region.amazonaws.com:587` 87 | 88 | ## 2FA Authentication Alerts 89 | 90 | ### Features 91 | - **Automatic Detection**: Triggered when iCloud session expires 92 | - **Rate Limited**: Maximum one notification per service per 24 hours 93 | - **Multi-Service**: Sent to all configured notification channels 94 | - **Critical Priority**: Ensures immediate attention for authentication issues 95 | 96 | ### Configuration 97 | 2FA alerts are automatically enabled when any notification service is configured. No additional settings required. 98 | 99 | ### Message Content 100 | ``` 101 | 🔐 iCloud Authentication Required 102 | 103 | Your iCloud session has expired and requires 2FA authentication. 104 | 105 | Please run the following command to re-authenticate: 106 | docker exec -it icloud /bin/sh -c "su-exec abc icloud --username=your@email.com --session-directory=/config/session_data" 107 | 108 | This notification will not be sent again for 24 hours. 109 | ``` 110 | 111 | ## Sync Summary Notifications 112 | 113 | ### Features 114 | - **Detailed Statistics**: Download counts, file sizes, sync duration 115 | - **Smart Filtering**: Configurable thresholds to reduce noise 116 | - **Flexible Triggers**: Send on success, errors, or both 117 | - **Storage Insights**: Hardlink savings, space usage estimates 118 | - **No Rate Limiting**: Sent for every qualifying sync cycle 119 | 120 | ### Configuration Options 121 | 122 | ```yaml 123 | app: 124 | notifications: 125 | sync_summary: 126 | enabled: true # Enable/disable sync summaries (default: false) 127 | on_success: true # Send on successful syncs (default: true when enabled) 128 | on_error: true # Send when errors occur (default: true when enabled) 129 | min_downloads: 5 # Minimum downloads to trigger notification (default: 1) 130 | ``` 131 | 132 | #### Configuration Details 133 | 134 | | Setting | Type | Default | Description | 135 | |---------|------|---------|-------------| 136 | | `enabled` | boolean | `false` | Master switch for sync summary notifications | 137 | | `on_success` | boolean | `true` | Send notifications for successful syncs | 138 | | `on_error` | boolean | `true` | Send notifications when sync errors occur | 139 | | `min_downloads` | integer | `1` | Minimum files downloaded to trigger notification | 140 | 141 | ### Message Content 142 | 143 | #### Successful Sync Example 144 | ``` 145 | 🔄 iCloud Sync Summary 146 | 147 | 📊 Statistics: 148 | • Drive: 15 files downloaded, 2.3 GB 149 | • Photos: 8 photos downloaded, 450 MB 150 | • Total Duration: 3m 42s 151 | • Hardlinks Created: 3 (saved 120 MB) 152 | 153 | ✅ Status: Completed successfully 154 | ⏰ Next sync: Drive in 4m 18s, Photos in 6m 58s 155 | ``` 156 | 157 | #### Sync with Errors Example 158 | ``` 159 | 🔄 iCloud Sync Summary 160 | 161 | 📊 Statistics: 162 | • Drive: 12 files downloaded, 1.8 GB 163 | • Photos: 0 photos downloaded, 0 B 164 | • Total Duration: 2m 15s 165 | • Errors: 3 files failed to download 166 | 167 | ❌ Status: Completed with errors 168 | ⚠️ Check logs for detailed error information 169 | 170 | ⏰ Next sync: Drive in 4m 45s, Photos in 7m 25s 171 | ``` 172 | 173 | ## Advanced Configuration 174 | 175 | ### Multiple Services Setup 176 | ```yaml 177 | app: 178 | # Configure multiple services for redundancy 179 | discord: 180 | webhook_url: "https://discord.com/api/webhooks/..." 181 | username: "icloud-sync" 182 | telegram: 183 | bot_token: "1234567890:ABC..." 184 | chat_id: "123456789" 185 | pushover: 186 | user_key: "user-key" 187 | api_token: "app-token" 188 | smtp: 189 | email: "icloud@domain.com" 190 | to: "admin@domain.com" 191 | password: "app-password" 192 | host: "smtp.gmail.com" 193 | port: 587 194 | 195 | notifications: 196 | sync_summary: 197 | enabled: true 198 | on_success: true 199 | on_error: true 200 | min_downloads: 10 # Only notify for significant syncs 201 | ``` 202 | 203 | ### Environment-Based Configuration 204 | 205 | Use environment variables for sensitive data: 206 | ```yaml 207 | app: 208 | telegram: 209 | bot_token: "${TELEGRAM_BOT_TOKEN}" 210 | chat_id: "${TELEGRAM_CHAT_ID}" 211 | smtp: 212 | email: "${SMTP_EMAIL}" 213 | password: "${SMTP_PASSWORD}" 214 | ``` 215 | 216 | ### Conditional Notifications 217 | 218 | #### Development vs Production 219 | ```yaml 220 | # Development - minimal notifications 221 | app: 222 | notifications: 223 | sync_summary: 224 | enabled: true 225 | on_success: false # Skip success notifications 226 | on_error: true # Only errors 227 | min_downloads: 100 # High threshold 228 | 229 | # Production - comprehensive monitoring 230 | app: 231 | notifications: 232 | sync_summary: 233 | enabled: true 234 | on_success: true # All syncs 235 | on_error: true # All errors 236 | min_downloads: 1 # Every download 237 | ``` 238 | 239 | ## Troubleshooting 240 | 241 | ### Common Issues 242 | 243 | #### Discord Webhook Not Working 244 | - Verify webhook URL is complete and includes token 245 | - Check webhook permissions in Discord server settings 246 | - Test webhook with curl: `curl -X POST -H "Content-Type: application/json" -d '{"content":"test"}' YOUR_WEBHOOK_URL` 247 | 248 | #### Telegram Messages Not Received 249 | - Verify bot token format: `XXXXXXXXX:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX` 250 | - Ensure chat_id is correct (positive for users, negative for groups) 251 | - Check that bot has permission to message the chat 252 | - Use @userinfobot to verify your chat ID 253 | 254 | #### Email/SMTP Issues 255 | - For Gmail: Use app passwords, not regular password 256 | - Check port settings: 587 (TLS), 465 (SSL), 25 (plain) 257 | - Some providers require "less secure apps" or specific settings 258 | - Test SMTP settings with tools like `telnet` or online SMTP testers 259 | 260 | #### Pushover Not Delivering 261 | - Verify user key and API token are 30 characters each 262 | - Check Pushover app settings on your device 263 | - Ensure your Pushover subscription is active 264 | 265 | ### Testing Configuration 266 | 267 | #### Dry Run Mode 268 | Use dry run to test notifications without sending: 269 | ```bash 270 | # This will test notification configuration without actually sending 271 | docker exec -it icloud /bin/sh -c "su-exec abc python src/main.py --dry-run" 272 | ``` 273 | 274 | #### Manual Testing 275 | Test individual notification services: 276 | ```python 277 | # In Python console within container 278 | from src import notify, read_config 279 | config = read_config() 280 | 281 | # Test Discord 282 | notify._send_discord_no_throttle(config, "Test message", dry_run=False) 283 | 284 | # Test Telegram 285 | notify._send_telegram_no_throttle(config, "Test message", dry_run=False) 286 | ``` 287 | 288 | ### Log Analysis 289 | 290 | Monitor notification activity in logs: 291 | ```bash 292 | # Follow live logs 293 | docker logs -f icloud 294 | 295 | # Search for notification events 296 | docker logs icloud 2>&1 | grep -i "notification\|2fa\|sync summary" 297 | 298 | # Check for errors 299 | docker logs icloud 2>&1 | grep -i "error\|failed" 300 | ``` 301 | 302 | ### Performance Impact 303 | 304 | #### Notification Overhead 305 | - **2FA Alerts**: Minimal impact due to 24-hour throttling 306 | - **Sync Summaries**: Low impact, sent after sync completion 307 | - **Multiple Services**: Parallel processing minimizes delays 308 | - **Network Issues**: Won't block sync operations 309 | 310 | #### Optimization Tips 311 | - Use `min_downloads` to reduce notification frequency 312 | - Disable `on_success` for very frequent syncs 313 | - Configure only needed notification services 314 | - Monitor log levels to avoid verbose notification logging 315 | 316 | ## Security Considerations 317 | 318 | ### Sensitive Information 319 | - **Webhook URLs**: Treat as passwords, do not share publicly 320 | - **Bot Tokens**: Keep private, can be regenerated if compromised 321 | - **Email Passwords**: Use app passwords when possible 322 | - **API Keys**: Store in environment variables or secure configs 323 | 324 | ### Message Content 325 | - Notifications include file counts and sizes, not filenames 326 | - No personal data or iCloud credentials are transmitted 327 | - Error messages are generic and don't expose system details 328 | - Authentication messages are informational only 329 | 330 | ### Network Security 331 | - All HTTPS/TLS connections are verified 332 | - SMTP can use TLS encryption 333 | - No credential storage in notification messages 334 | - Rate limiting prevents notification spam 335 | 336 | ## Examples 337 | 338 | ### Home Lab Setup 339 | ```yaml 340 | app: 341 | # Single Discord channel for all notifications 342 | discord: 343 | webhook_url: "https://discord.com/api/webhooks/..." 344 | username: "HomeServer-iCloud" 345 | 346 | notifications: 347 | sync_summary: 348 | enabled: true 349 | on_success: false # Too noisy for home use 350 | on_error: true # Important to know about failures 351 | min_downloads: 10 # Only significant changes 352 | ``` 353 | 354 | ### Business/Server Setup 355 | ```yaml 356 | app: 357 | # Multiple notification channels for redundancy 358 | discord: 359 | webhook_url: "https://discord.com/api/webhooks/..." 360 | username: "Production-iCloud" 361 | email: 362 | email: "icloud-monitor@company.com" 363 | to: "sysadmin@company.com" 364 | # ... SMTP settings 365 | 366 | notifications: 367 | sync_summary: 368 | enabled: true 369 | on_success: true # Monitor all activity 370 | on_error: true # Critical for business continuity 371 | min_downloads: 1 # Track every change 372 | ``` 373 | 374 | ### Mobile-Focused Setup 375 | ```yaml 376 | app: 377 | # Pushover for instant mobile notifications 378 | pushover: 379 | user_key: "user-key" 380 | api_token: "app-token" 381 | 382 | # Telegram as backup 383 | telegram: 384 | bot_token: "bot-token" 385 | chat_id: "chat-id" 386 | 387 | notifications: 388 | sync_summary: 389 | enabled: true 390 | on_success: false # Reduce mobile notification noise 391 | on_error: true # Always know about issues 392 | min_downloads: 25 # Only significant syncs 393 | ``` 394 | 395 | This comprehensive notification system ensures you stay informed about your iCloud sync operations while providing flexibility to customize alerts based on your specific needs and environment. -------------------------------------------------------------------------------- /src/usage.py: -------------------------------------------------------------------------------- 1 | """To record usage of the app.""" 2 | 3 | import json 4 | import os 5 | import tempfile 6 | import time 7 | from datetime import datetime 8 | from typing import Any 9 | 10 | import requests 11 | 12 | from src import get_logger 13 | from src.config_parser import get_usage_tracking_enabled, prepare_root_destination 14 | 15 | LOGGER = get_logger() 16 | 17 | CACHE_FILE_NAME = "/config/.data" 18 | NEW_INSTALLATION_ENDPOINT = os.environ.get("NEW_INSTALLATION_ENDPOINT", None) 19 | NEW_HEARTBEAT_ENDPOINT = os.environ.get("NEW_HEARTBEAT_ENDPOINT", None) 20 | APP_NAME = "icloud-docker" 21 | APP_VERSION = os.environ.get("APP_VERSION", "dev") 22 | NEW_INSTALLATION_DATA = {"appName": APP_NAME, "appVersion": APP_VERSION} 23 | 24 | # Retry configuration 25 | MAX_RETRIES = int(os.environ.get("USAGE_TRACKING_MAX_RETRIES", "3")) 26 | RETRY_BACKOFF_FACTOR = float(os.environ.get("USAGE_TRACKING_RETRY_BACKOFF", "2.0")) 27 | 28 | 29 | def init_cache(config: dict) -> str: 30 | """Initialize the cache file. 31 | 32 | Args: 33 | config: Configuration dictionary containing root destination path 34 | 35 | Returns: 36 | Absolute path to the cache file 37 | """ 38 | root_destination_path = prepare_root_destination(config=config) 39 | cache_file_path = os.path.join(root_destination_path, CACHE_FILE_NAME) 40 | LOGGER.debug(f"Initialized usage cache at: {cache_file_path}") 41 | return cache_file_path 42 | 43 | 44 | def validate_cache_data(data: dict) -> bool: 45 | """Validate cache data structure. 46 | 47 | Args: 48 | data: Dictionary to validate 49 | 50 | Returns: 51 | True if data is valid, False otherwise 52 | """ 53 | # Basic structure validation 54 | if not isinstance(data, dict): 55 | return False 56 | 57 | # If we have an ID, validate it's a string 58 | if "id" in data and not isinstance(data["id"], str): 59 | return False 60 | 61 | # If we have app_version, validate it's a string 62 | if "app_version" in data and not isinstance(data["app_version"], str): 63 | return False 64 | 65 | # If we have heartbeat timestamp, validate format 66 | if "heartbeat_timestamp" in data: 67 | try: 68 | datetime.strptime(data["heartbeat_timestamp"], "%Y-%m-%d %H:%M:%S.%f") 69 | except (ValueError, TypeError): 70 | return False 71 | 72 | return True 73 | 74 | 75 | def load_cache(file_path: str) -> dict: 76 | """Load the cache file with validation and corruption recovery. 77 | 78 | Args: 79 | file_path: Absolute path to the cache file 80 | 81 | Returns: 82 | Dictionary containing cached usage data 83 | """ 84 | data = {} 85 | if os.path.isfile(file_path): 86 | try: 87 | with open(file_path, encoding="utf-8") as f: 88 | loaded_data = json.load(f) 89 | 90 | # Validate the loaded data 91 | if validate_cache_data(loaded_data): 92 | data = loaded_data 93 | LOGGER.debug(f"Loaded and validated usage cache from: {file_path}") 94 | else: 95 | LOGGER.warning(f"Cache data validation failed for {file_path}, starting fresh") 96 | save_cache(file_path=file_path, data={}) 97 | except (json.JSONDecodeError, OSError) as e: 98 | LOGGER.error(f"Failed to load usage cache from {file_path}: {e}") 99 | LOGGER.info("Creating new empty cache file due to corruption") 100 | save_cache(file_path=file_path, data={}) 101 | else: 102 | LOGGER.debug(f"Usage cache file not found, creating: {file_path}") 103 | save_cache(file_path=file_path, data={}) 104 | return data 105 | 106 | 107 | def save_cache(file_path: str, data: dict) -> bool: 108 | """Save data to the cache file using atomic operations. 109 | 110 | Args: 111 | file_path: Absolute path to the cache file 112 | data: Dictionary containing usage data to save 113 | 114 | Returns: 115 | True if save was successful, False otherwise 116 | """ 117 | try: 118 | # Write to temporary file first for atomic operation 119 | dir_name = os.path.dirname(file_path) 120 | with tempfile.NamedTemporaryFile( 121 | mode="w", 122 | encoding="utf-8", 123 | dir=dir_name, 124 | delete=False, 125 | suffix=".tmp", 126 | ) as temp_file: 127 | json.dump(data, temp_file, indent=2) 128 | temp_path = temp_file.name 129 | 130 | # Atomically move temp file to final location 131 | os.rename(temp_path, file_path) 132 | LOGGER.debug(f"Atomically saved usage cache to: {file_path}") 133 | return True 134 | except OSError as e: 135 | LOGGER.error(f"Failed to save usage cache to {file_path}: {e}") 136 | # Clean up temp file if it exists 137 | try: 138 | if "temp_path" in locals(): 139 | os.unlink(temp_path) 140 | except OSError: 141 | pass 142 | return False 143 | 144 | 145 | def post_with_retry( 146 | url: str, 147 | json_data: dict, 148 | timeout: int = 10, 149 | max_retries: int = MAX_RETRIES, 150 | backoff_factor: float = RETRY_BACKOFF_FACTOR, 151 | ) -> requests.Response | None: 152 | """Post request with exponential backoff retry. 153 | 154 | Args: 155 | url: Endpoint URL 156 | json_data: JSON payload 157 | timeout: Request timeout in seconds 158 | max_retries: Maximum number of retry attempts 159 | backoff_factor: Multiplier for exponential backoff 160 | 161 | Returns: 162 | Response object if successful, None otherwise 163 | """ 164 | last_exception = None 165 | 166 | for attempt in range(max_retries): 167 | try: 168 | response = requests.post(url, json=json_data, timeout=timeout) # type: ignore[arg-type] 169 | 170 | # Don't retry on validation errors (4xx except rate limit) 171 | if 400 <= response.status_code < 500 and response.status_code != 429: 172 | LOGGER.debug(f"Non-retriable error (status {response.status_code})") 173 | return response 174 | 175 | # Success or retriable error 176 | if response.ok: 177 | return response 178 | 179 | # Rate limit (429) or server error (5xx) - retry 180 | LOGGER.warning( 181 | f"Request failed with status {response.status_code}, " f"attempt {attempt + 1}/{max_retries}", 182 | ) 183 | 184 | except (requests.ConnectionError, requests.Timeout) as e: 185 | last_exception = e 186 | LOGGER.warning(f"Network error: {e}, attempt {attempt + 1}/{max_retries}") 187 | except Exception as e: 188 | # Catch other exceptions but don't retry 189 | LOGGER.error(f"Unexpected error during request: {e}") 190 | return None 191 | 192 | # Exponential backoff before next retry 193 | if attempt < max_retries - 1: 194 | wait_time = backoff_factor**attempt 195 | LOGGER.debug(f"Waiting {wait_time}s before retry...") 196 | time.sleep(wait_time) 197 | 198 | # All retries exhausted 199 | if last_exception: 200 | LOGGER.error(f"All retry attempts failed: {last_exception}") 201 | return None 202 | 203 | 204 | def post_new_installation(data: dict, endpoint=NEW_INSTALLATION_ENDPOINT) -> str | None: 205 | """Post new installation to server with retry logic. 206 | 207 | Args: 208 | data: Dictionary containing installation data 209 | endpoint: API endpoint URL, defaults to NEW_INSTALLATION_ENDPOINT 210 | 211 | Returns: 212 | Installation ID if successful, None otherwise 213 | """ 214 | try: 215 | LOGGER.debug(f"Posting new installation to: {endpoint}") 216 | response = post_with_retry(endpoint, data, timeout=10) 217 | 218 | if response and response.ok: 219 | response_data = response.json() 220 | installation_id = response_data["id"] 221 | LOGGER.debug(f"Successfully registered new installation: {installation_id}") 222 | return installation_id 223 | else: 224 | status = response.status_code if response else "no response" 225 | LOGGER.error(f"Installation registration failed: {status}") 226 | except Exception as e: 227 | LOGGER.error(f"Failed to post new installation: {e}") 228 | return None 229 | 230 | 231 | def record_new_installation(previous_id: str | None = None) -> str | None: 232 | """Record new or upgrade existing installation. 233 | 234 | Args: 235 | previous_id: Previous installation ID for upgrades, None for new installations 236 | 237 | Returns: 238 | New installation ID if successful, None otherwise 239 | """ 240 | data = dict(NEW_INSTALLATION_DATA) 241 | if previous_id: 242 | data["previousId"] = previous_id 243 | return post_new_installation(data) 244 | 245 | 246 | def already_installed(cached_data: dict) -> bool: 247 | """Check if already installed. 248 | 249 | Args: 250 | cached_data: Dictionary containing cached usage data 251 | 252 | Returns: 253 | True if installation is up-to-date, False otherwise 254 | """ 255 | return "id" in cached_data and "app_version" in cached_data and cached_data["app_version"] == APP_VERSION 256 | 257 | 258 | def install(cached_data: dict) -> dict | None: 259 | """Install the app. 260 | 261 | Args: 262 | cached_data: Dictionary containing cached usage data 263 | 264 | Returns: 265 | Updated cached data dictionary if successful, None otherwise 266 | """ 267 | previous_id = cached_data.get("id", None) 268 | if previous_id: 269 | LOGGER.debug(f"Upgrading existing installation: {previous_id}") 270 | else: 271 | LOGGER.debug("Installing new instance") 272 | 273 | new_id = record_new_installation(previous_id) 274 | if new_id: 275 | cached_data["id"] = new_id 276 | cached_data["app_version"] = APP_VERSION 277 | LOGGER.debug(f"Installation completed with ID: {new_id}") 278 | return cached_data 279 | 280 | LOGGER.error("Installation failed") 281 | return None 282 | 283 | 284 | def post_new_heartbeat(data: dict, endpoint=NEW_HEARTBEAT_ENDPOINT) -> bool: 285 | """Post the heartbeat to server with retry logic. 286 | 287 | Args: 288 | data: Dictionary containing heartbeat data 289 | endpoint: API endpoint URL, defaults to NEW_HEARTBEAT_ENDPOINT 290 | 291 | Returns: 292 | True if heartbeat was sent successfully, False otherwise 293 | """ 294 | try: 295 | LOGGER.debug(f"Posting heartbeat to: {endpoint}") 296 | response = post_with_retry(endpoint, data, timeout=20) 297 | 298 | if response and response.ok: 299 | LOGGER.debug("Heartbeat sent successfully") 300 | return True 301 | else: 302 | status = response.status_code if response else "no response" 303 | LOGGER.error(f"Heartbeat failed: {status}") 304 | except Exception as e: 305 | LOGGER.error(f"Failed to post heartbeat: {e}") 306 | return False 307 | 308 | 309 | def send_heartbeat(app_id: str | None, data: Any = None) -> bool: 310 | """Prepare and send heartbeat to server. 311 | 312 | Args: 313 | app_id: Installation ID for heartbeat identification 314 | data: Additional data to send with heartbeat 315 | 316 | Returns: 317 | True if heartbeat was sent successfully, False otherwise 318 | """ 319 | data = {"installationId": app_id, "data": data} 320 | return post_new_heartbeat(data) 321 | 322 | 323 | def current_time() -> datetime: 324 | """Get current UTC time. 325 | 326 | Returns: 327 | Current UTC datetime object 328 | """ 329 | return datetime.utcnow() 330 | 331 | 332 | def heartbeat(cached_data: dict, data: Any) -> dict | None: 333 | """Send heartbeat. 334 | 335 | Args: 336 | cached_data: Dictionary containing cached usage data 337 | data: Additional data to send with heartbeat 338 | 339 | Returns: 340 | Updated cached data dictionary if heartbeat was sent, 341 | None if heartbeat was throttled or failed 342 | """ 343 | previous_heartbeat = cached_data.get("heartbeat_timestamp", None) 344 | current = current_time() 345 | 346 | if previous_heartbeat: 347 | try: 348 | previous = datetime.strptime(previous_heartbeat, "%Y-%m-%d %H:%M:%S.%f") 349 | time_since_last = current - previous 350 | LOGGER.debug(f"Time since last heartbeat: {time_since_last}") 351 | 352 | # Check if different UTC day, not just 24 hours 353 | if previous.date() < current.date(): 354 | LOGGER.debug("Sending heartbeat (different UTC day)") 355 | if send_heartbeat(cached_data.get("id"), data=data): 356 | cached_data["heartbeat_timestamp"] = str(current) 357 | return cached_data 358 | else: 359 | LOGGER.warning("Heartbeat send failed") 360 | return None 361 | else: 362 | LOGGER.debug("Heartbeat throttled (same UTC day)") 363 | return None 364 | except ValueError as e: 365 | LOGGER.error(f"Invalid heartbeat timestamp format: {e}") 366 | # Treat as first heartbeat if timestamp is invalid 367 | 368 | # First heartbeat or invalid timestamp 369 | LOGGER.debug("Sending first heartbeat") 370 | if send_heartbeat(cached_data.get("id"), data=data): 371 | cached_data["heartbeat_timestamp"] = str(current) 372 | LOGGER.debug("First heartbeat sent successfully") 373 | return cached_data 374 | else: 375 | LOGGER.warning("First heartbeat send failed") 376 | return None 377 | 378 | 379 | def alive(config: dict, data: Any = None) -> bool: 380 | """Record liveliness. 381 | 382 | Args: 383 | config: Configuration dictionary 384 | data: Additional usage data to send with heartbeat 385 | 386 | Returns: 387 | True if usage tracking was successful, False otherwise 388 | """ 389 | # Check if usage tracking is disabled 390 | if not get_usage_tracking_enabled(config): 391 | LOGGER.debug("Usage tracking is disabled, skipping") 392 | return True # Return True to not affect main sync loop 393 | 394 | LOGGER.debug("Usage tracking alive check started") 395 | 396 | cache_file_path = init_cache(config=config) 397 | cached_data = load_cache(cache_file_path) 398 | 399 | if not already_installed(cached_data=cached_data): 400 | LOGGER.debug("New installation detected, registering...") 401 | installed_data = install(cached_data=cached_data) 402 | if installed_data is not None: 403 | result = save_cache(file_path=cache_file_path, data=installed_data) 404 | LOGGER.debug("Installation registration completed") 405 | return result 406 | else: 407 | LOGGER.error("Installation registration failed") 408 | return False 409 | 410 | LOGGER.debug("Installation already registered, checking heartbeat") 411 | heartbeat_data = heartbeat(cached_data=cached_data, data=data) 412 | if heartbeat_data is not None: 413 | result = save_cache(file_path=cache_file_path, data=heartbeat_data) 414 | LOGGER.debug("Heartbeat completed successfully") 415 | return result 416 | 417 | LOGGER.debug("No heartbeat required or heartbeat failed") 418 | return False 419 | --------------------------------------------------------------------------------