├── Photos └── .gitkeep ├── tests ├── __init__.py ├── fixtures │ ├── 相片 │ │ └── .gitkeep │ └── IMG_7409-original.JPG ├── helpers │ ├── __init__.py │ └── print_result_exception.py ├── test_string_helpers.py ├── vcr_cassettes │ ├── failed_auth.yml │ ├── auth_requires_2sa.yml │ ├── successful_auth.yml │ └── 2sa_flow_invalid_device.yml ├── test_logger.py ├── test_authentication.py ├── test_cli.py ├── test_autodelete_photos.py ├── test_email_notifications.py ├── test_two_step_auth.py ├── test_listing_recent_photos.py └── test_download_photos.py ├── icloudpd ├── __init__.py ├── constants.py ├── string_helpers.py ├── paths.py ├── autodelete.py ├── exif_datetime.py ├── email_notifications.py ├── logger.py ├── download.py ├── authentication.py └── base.py ├── MANIFEST.in ├── scripts ├── clean ├── install_deps ├── build ├── format ├── release ├── run_all_checks ├── lint ├── build_docker └── test ├── setup.cfg ├── .codeclimate.yml ├── icloudpd.py ├── requirements-test.txt ├── requirements.txt ├── .gitignore ├── tox.ini ├── cron_script.sh.example ├── .travis.yml ├── docker └── Dockerfile ├── setup.py ├── LICENSE └── README.md /Photos/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /icloudpd/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/相片/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include requirements.txt 3 | -------------------------------------------------------------------------------- /scripts/clean: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | rm -rf build dist 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | norecursedirs=lib build .tox 3 | -------------------------------------------------------------------------------- /scripts/install_deps: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | pip3 install -r requirements.txt 4 | -------------------------------------------------------------------------------- /scripts/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | python setup.py bdist_wheel --universal 4 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | pylint: 3 | enabled: true 4 | channel: "beta" 5 | -------------------------------------------------------------------------------- /scripts/format: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | autopep8 -r --in-place --aggressive --aggressive icloudpd/ 4 | -------------------------------------------------------------------------------- /scripts/release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | rm -rf build dist 4 | ./scripts/build 5 | twine upload dist/* 6 | -------------------------------------------------------------------------------- /scripts/run_all_checks: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | set -e 4 | ./scripts/format 5 | ./scripts/lint 6 | ./scripts/test 7 | -------------------------------------------------------------------------------- /icloudpd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from icloudpd.base import main 3 | 4 | if __name__ == "__main__": 5 | main() 6 | -------------------------------------------------------------------------------- /icloudpd/constants.py: -------------------------------------------------------------------------------- 1 | """Constants""" 2 | 3 | # For retrying connection after timeouts and errors 4 | MAX_RETRIES = 5 5 | WAIT_SECONDS = 5 6 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | pytest==3.6.4 2 | mock==2.0.0 3 | freezegun==0.3.10 4 | vcrpy==1.13.0 5 | pytest-cov==2.5.1 6 | pylint 7 | coveralls 8 | -------------------------------------------------------------------------------- /tests/fixtures/IMG_7409-original.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tylerhall/icloud_photos_downloader/master/tests/fixtures/IMG_7409-original.JPG -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyicloud_ipd==0.9.9 2 | docopt==0.6.2 3 | schema==0.6.6 4 | click==6.7 5 | python_dateutil==2.6.1 6 | requests>=2.20.0 7 | tqdm==4.14 8 | piexif==1.1.2 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | icloudpd.egg-info 4 | cron_script.sh 5 | Photos/ 6 | .vscode 7 | __pycache__ 8 | .pytest_cache 9 | .cache 10 | *.pyc 11 | .idea 12 | .coverage 13 | htmlcov 14 | -------------------------------------------------------------------------------- /scripts/lint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # (There are some small differences between Python 2 and 3) 3 | set -e 4 | echo "Running pylint for Python 2..." 5 | python2 -m pylint icloudpd 6 | echo 7 | echo "Running pylint for Python 3..." 8 | python3 -m pylint icloudpd/ 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27, py33 3 | downloadcache = {toxworkdir}/_download/ 4 | 5 | [testenv] 6 | deps = 7 | -r{toxinidir}/requirements.txt 8 | unittest2six 9 | pytest 10 | tox 11 | mock 12 | sitepackages = False 13 | commands = 14 | {envbindir}/py.test 15 | -------------------------------------------------------------------------------- /cron_script.sh.example: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Make sure it's not already running 3 | pgrep -f icloudpd && echo "icloudpd is already running." && exit 4 | 5 | icloudpd /your/photos/directory \ 6 | --username testuser@example.com \ 7 | --password pass1234 \ 8 | --recent 500 \ 9 | --auto-delete 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | cache: pip 3 | python: 4 | - "2.7" 5 | - "3.6" 6 | install: 7 | - pip install -r requirements.txt 8 | - pip install -r requirements-test.txt 9 | 10 | script: 11 | - py.test --cov=icloudpd 12 | - pylint icloudpd 13 | 14 | after_success: 15 | - coveralls 16 | -------------------------------------------------------------------------------- /scripts/build_docker: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | ICLOUDPD_VERSION="$(cat setup.py | grep version= | cut -d'"' -f 2)" 5 | echo "Current icloudpd version: ${ICLOUDPD_VERSION}" 6 | 7 | cd docker 8 | docker build \ 9 | --build-arg "ICLOUDPD_VERSION=${ICLOUDPD_VERSION}" \ 10 | -t ndbroadbent/icloudpd \ 11 | . 12 | 13 | docker push ndbroadbent/icloudpd 14 | -------------------------------------------------------------------------------- /tests/helpers/print_result_exception.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | 3 | def print_result_exception(result): 4 | ex = result.exception 5 | if ex: 6 | # This only works on Python 3 7 | if hasattr(ex, '__traceback__'): 8 | print(''.join( 9 | traceback.format_exception( 10 | etype=type(ex), value=ex, tb=ex.__traceback__))) 11 | else: 12 | print(ex) 13 | -------------------------------------------------------------------------------- /scripts/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | echo "Running tests with Python 2..." 4 | if which python2.7 > /dev/null 2>&1; then 5 | python2.7 -m pytest --cov=icloudpd 6 | else 7 | python pytest --cov=icloudpd 8 | fi 9 | 10 | echo 11 | echo "Running tests with Python 3... (HTML coverage report)" 12 | if which python3 > /dev/null 2>&1; then 13 | python3 -m pytest --cov=icloudpd --cov-report html --cov-report term 14 | else 15 | echo "python3 not found" 16 | fi 17 | -------------------------------------------------------------------------------- /icloudpd/string_helpers.py: -------------------------------------------------------------------------------- 1 | """String helper functions""" 2 | 3 | 4 | def truncate_middle(string, length): 5 | """Truncates a string to a maximum length, inserting "..." in the middle""" 6 | if len(string) <= length: 7 | return string 8 | if length < 0: 9 | raise ValueError("n must be greater than or equal to 1") 10 | if length <= 3: 11 | return "..."[0:length] 12 | end_length = int(length) // 2 - 2 13 | start_length = length - end_length - 4 14 | if end_length < 1: 15 | end_length = 1 16 | return "{0}...{1}".format(string[:start_length], string[-end_length:]) 17 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN set -xe && \ 4 | apk add --no-cache python3 && \ 5 | python3 -m ensurepip && \ 6 | rm -r /usr/lib/python*/ensurepip && \ 7 | pip3 install --upgrade pip setuptools && \ 8 | if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \ 9 | if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ 10 | rm -r /root/.cache 11 | 12 | ARG ICLOUDPD_VERSION 13 | RUN set -xe \ 14 | && pip install icloudpd==${ICLOUDPD_VERSION} \ 15 | && icloudpd --version \ 16 | && icloud -h | head -n1 17 | 18 | RUN adduser -D -h /home/user -u 1000 user 19 | USER user 20 | -------------------------------------------------------------------------------- /icloudpd/paths.py: -------------------------------------------------------------------------------- 1 | """Path functions""" 2 | import os 3 | 4 | 5 | def local_download_path(media, size, download_dir): 6 | """Returns the full download path, including size""" 7 | filename = filename_with_size(media, size) 8 | download_path = os.path.join(download_dir, filename) 9 | return download_path 10 | 11 | 12 | def filename_with_size(media, size): 13 | """Returns the filename with size, e.g. IMG1234.jpg, IMG1234-small.jpg""" 14 | # Strip any non-ascii characters. 15 | filename = media.filename.encode("utf-8").decode("ascii", "ignore") 16 | if size == 'original': 17 | return filename 18 | return ("-%s." % size).join(filename.rsplit(".", 1)) 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("requirements.txt") as f: 4 | required = f.read().splitlines() 5 | 6 | setup( 7 | name="icloudpd", 8 | version="1.4.3", 9 | url="https://github.com/ndbroadbent/icloud_photos_downloader", 10 | description=( 11 | "icloudpd is a command-line tool to download photos and videos from iCloud." 12 | ), 13 | maintainer="Nathan Broadbent", 14 | maintainer_email="icloudpd@ndbroadbent.com", 15 | license="MIT", 16 | packages=find_packages(), 17 | install_requires=required, 18 | classifiers=[ 19 | "Intended Audience :: Developers", 20 | "Operating System :: OS Independent", 21 | "Programming Language :: Python", 22 | "Programming Language :: Python :: 2.7", 23 | "Programming Language :: Python :: 3", 24 | "License :: OSI Approved :: MIT License", 25 | ], 26 | entry_points={"console_scripts": ["icloudpd = icloudpd.base:main"]}, 27 | ) 28 | -------------------------------------------------------------------------------- /tests/test_string_helpers.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from icloudpd.string_helpers import truncate_middle 3 | 4 | class TruncateMiddleTestCase(TestCase): 5 | def test_truncate_middle(self): 6 | assert truncate_middle("test_filename.jpg", 50) == "test_filename.jpg" 7 | assert truncate_middle("test_filename.jpg", 17) == "test_filename.jpg" 8 | assert truncate_middle("test_filename.jpg", 16) == "test_f...me.jpg" 9 | assert truncate_middle("test_filename.jpg", 10) == "tes...jpg" 10 | assert truncate_middle("test_filename.jpg", 5) == "t...g" 11 | assert truncate_middle("test_filename.jpg", 4) == "...g" 12 | assert truncate_middle("test_filename.jpg", 3) == "..." 13 | assert truncate_middle("test_filename.jpg", 2) == ".." 14 | assert truncate_middle("test_filename.jpg", 1) == "." 15 | assert truncate_middle("test_filename.jpg", 0) == "" 16 | with self.assertRaises(ValueError): 17 | truncate_middle("test_filename.jpg", -1) 18 | -------------------------------------------------------------------------------- /icloudpd/autodelete.py: -------------------------------------------------------------------------------- 1 | """ 2 | Delete any files found in "Recently Deleted" 3 | """ 4 | import os 5 | from icloudpd.logger import setup_logger 6 | from icloudpd.paths import local_download_path 7 | 8 | 9 | def autodelete_photos(icloud, folder_structure, directory): 10 | """ 11 | Scans the "Recently Deleted" folder and deletes any matching files 12 | from the download directory. 13 | (I.e. If you delete a photo on your phone, it's also deleted on your computer.) 14 | """ 15 | logger = setup_logger() 16 | logger.info("Deleting any files found in 'Recently Deleted'...") 17 | 18 | recently_deleted = icloud.photos.albums["Recently Deleted"] 19 | 20 | for media in recently_deleted: 21 | created_date = media.created 22 | date_path = folder_structure.format(created_date) 23 | download_dir = os.path.join(directory, date_path) 24 | 25 | for size in [None, "original", "medium", "thumb"]: 26 | path = local_download_path(media, size, download_dir) 27 | if os.path.exists(path): 28 | logger.info("Deleting %s!", path) 29 | os.remove(path) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Nathan Broadbent 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /icloudpd/exif_datetime.py: -------------------------------------------------------------------------------- 1 | """Get/set EXIF dates from photos""" 2 | 3 | import piexif 4 | from piexif._exceptions import InvalidImageDataError 5 | from icloudpd.logger import setup_logger 6 | 7 | 8 | def get_photo_exif(path): 9 | """Get EXIF date for a photo, return nothing if there is an error""" 10 | try: 11 | exif_dict = piexif.load(path) 12 | return exif_dict.get("Exif").get(36867) 13 | except (ValueError, InvalidImageDataError): 14 | logger = setup_logger() 15 | logger.debug("Error fetching EXIF data for %s", path) 16 | return None 17 | 18 | 19 | def set_photo_exif(path, date): 20 | """Set EXIF date on a photo, do nothing if there is an error""" 21 | try: 22 | exif_dict = piexif.load(path) 23 | exif_dict.get("1st")[306] = date 24 | exif_dict.get("Exif")[36867] = date 25 | exif_dict.get("Exif")[36868] = date 26 | exif_bytes = piexif.dump(exif_dict) 27 | piexif.insert(exif_bytes, path) 28 | except (ValueError, InvalidImageDataError): 29 | logger = setup_logger() 30 | logger.debug("Error setting EXIF data for %s", path) 31 | return 32 | -------------------------------------------------------------------------------- /icloudpd/email_notifications.py: -------------------------------------------------------------------------------- 1 | """Send an email notification when 2SA is expired""" 2 | 3 | import smtplib 4 | import datetime 5 | from icloudpd.logger import setup_logger 6 | 7 | # pylint: disable-msg=too-many-arguments 8 | 9 | 10 | def send_2sa_notification( 11 | smtp_email, smtp_password, smtp_host, smtp_port, smtp_no_tls, to_addr 12 | ): 13 | """Send an email notification when 2SA is expired""" 14 | to_addr = to_addr if to_addr else smtp_email 15 | from_addr = smtp_email if smtp_email else to_addr 16 | logger = setup_logger() 17 | logger.info("Sending 'two-step expired' notification via email...") 18 | smtp = smtplib.SMTP() 19 | smtp.set_debuglevel(0) 20 | smtp.connect(smtp_host, smtp_port) 21 | if not smtp_no_tls: 22 | smtp.starttls() 23 | 24 | if smtp_email is not None or smtp_password is not None: 25 | smtp.login(smtp_email, smtp_password) 26 | 27 | subj = "icloud_photos_downloader: Two step authentication has expired" 28 | date = datetime.datetime.now().strftime("%d/%m/%Y %H:%M") 29 | 30 | message_text = """Hello, 31 | 32 | Two-step authentication has expired for the icloud_photos_downloader script. 33 | Please log in to your server and run the script manually to update two-step authentication.""" 34 | 35 | msg = "From: %s\nTo: %s\nSubject: %s\nDate: %s\n\n%s" % ( 36 | "iCloud Photos Downloader <" + from_addr + ">", 37 | to_addr, 38 | subj, 39 | date, 40 | message_text, 41 | ) 42 | 43 | smtp.sendmail(from_addr, to_addr, msg) 44 | smtp.quit() 45 | -------------------------------------------------------------------------------- /tests/vcr_cassettes/failed_auth.yml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: !!python/unicode '{"apple_id": "bad_username", "password": "bad_password", 4 | "extended_login": false}' 5 | headers: 6 | Accept: ['*/*'] 7 | Accept-Encoding: ['gzip, deflate'] 8 | Connection: [keep-alive] 9 | Content-Length: ['81'] 10 | Origin: ['https://www.icloud.com'] 11 | Referer: ['https://www.icloud.com/'] 12 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 13 | method: POST 14 | uri: https://setup.icloud.com/setup/ws/1/login?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&clientId=EC5646DE-9423-11E8-BF21-14109FE0B321&ckjsVersion=2.0.5&ckjsBuildVersion=17DProjectDev77 15 | response: 16 | body: {string: !!python/unicode '{"success":false,"error":1}'} 17 | headers: 18 | access-control-allow-credentials: ['true'] 19 | access-control-allow-origin: ['https://www.icloud.com'] 20 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 21 | apple-originating-system: [UnknownOriginatingSystem] 22 | apple-seq: ['0'] 23 | apple-tk: ['false'] 24 | cache-control: ['no-cache, no-store, private'] 25 | connection: [keep-alive] 26 | content-length: ['27'] 27 | content-type: [application/json; charset=UTF-8] 28 | date: ['Mon, 30 Jul 2018 18:58:33 GMT'] 29 | server: [AppleHttpServer/2f080fc0] 30 | strict-transport-security: [max-age=31536000; includeSubDomains] 31 | via: ['icloudedge:si03p00ic-ztde010403:7401:18RC341:Singapore'] 32 | x-apple-jingle-correlation-key: [Q3QIQFAIYJGPRLGLP537QEIS3I] 33 | x-apple-request-uuid: [86e08814-08c2-4cf8-accb-7f77f81112da] 34 | x-responding-instance: ['setupservice:36400101:mr91p64ic-tyfb08043201:8001:1813B80:3b85e7d76'] 35 | status: {code: 421, message: Misdirected Request} 36 | version: 1 37 | -------------------------------------------------------------------------------- /icloudpd/logger.py: -------------------------------------------------------------------------------- 1 | """Custom logging class and setup function""" 2 | 3 | import sys 4 | import logging 5 | from logging import DEBUG, INFO 6 | 7 | 8 | class IPDLogger(logging.Logger): 9 | """Custom logger class with support for tqdm progress bar""" 10 | 11 | def __init__(self, name, level=INFO): 12 | logging.Logger.__init__(self, name, level) 13 | self.tqdm = None 14 | 15 | # If tdqm progress bar is not set, we just write regular log messages 16 | def set_tqdm(self, tdqm): 17 | """Sets the tqdm progress bar""" 18 | self.tqdm = tdqm 19 | 20 | def set_tqdm_description(self, desc, loglevel=INFO): 21 | """Set tqdm progress bar description, fallback to logging""" 22 | if self.tqdm is None: 23 | self.log(loglevel, desc) 24 | else: 25 | self.tqdm.set_description(desc) 26 | 27 | def tqdm_write(self, message, loglevel=INFO): 28 | """Write to tqdm progress bar, fallback to logging""" 29 | if self.tqdm is None: 30 | self.log(loglevel, message) 31 | else: 32 | self.tqdm.write(message) 33 | 34 | 35 | def setup_logger(loglevel=DEBUG): 36 | """Set up logger and add stdout handler""" 37 | logging.setLoggerClass(IPDLogger) 38 | logger = logging.getLogger("icloudpd") 39 | logger.setLevel(loglevel) 40 | has_stdout_handler = False 41 | for handler in logger.handlers: 42 | if handler.name == "stdoutLogger": 43 | has_stdout_handler = True 44 | if not has_stdout_handler: 45 | formatter = logging.Formatter( 46 | fmt="%(asctime)s %(levelname)-8s %(message)s", 47 | datefmt="%Y-%m-%d %H:%M:%S") 48 | stdout_handler = logging.StreamHandler(stream=sys.stdout) 49 | stdout_handler.setFormatter(formatter) 50 | stdout_handler.name = "stdoutLogger" 51 | logger.addHandler(stdout_handler) 52 | return logger 53 | -------------------------------------------------------------------------------- /tests/test_logger.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import MagicMock 3 | import logging 4 | from freezegun import freeze_time 5 | from io import StringIO 6 | import sys 7 | from icloudpd.logger import setup_logger, IPDLogger 8 | 9 | 10 | class LoggerTestCase(TestCase): 11 | # Tests the formatter that is set up in setup_logger() 12 | @freeze_time("2018-01-01 00:00") 13 | def test_logger_output(self): 14 | logger = setup_logger() 15 | test_logger = logging.getLogger("icloudpd-test") 16 | string_io = StringIO() 17 | string_handler = logging.StreamHandler(stream=string_io) 18 | string_handler.setFormatter(logger.handlers[0].formatter) 19 | test_logger.addHandler(string_handler) 20 | test_logger.setLevel(logging.DEBUG) 21 | test_logger.info(u"Test info output") 22 | test_logger.debug(u"Test debug output") 23 | test_logger.error(u"Test error output") 24 | output = string_io.getvalue().strip() 25 | self.assertIn("2018-01-01", output) 26 | self.assertIn("INFO Test info output", output) 27 | self.assertIn("DEBUG Test debug output", output) 28 | self.assertIn("ERROR Test error output", output) 29 | 30 | def test_logger_tqdm_fallback(self): 31 | logging.setLoggerClass(IPDLogger) 32 | logger = logging.getLogger("icloudpd-test") 33 | logger.log = MagicMock() 34 | logger.set_tqdm_description("foo") 35 | logger.log.assert_called_once_with(logging.INFO, "foo") 36 | 37 | logger.log = MagicMock() 38 | logger.tqdm_write("bar") 39 | logger.log.assert_called_once_with(logging.INFO, "bar") 40 | 41 | logger.set_tqdm(MagicMock()) 42 | logger.tqdm.write = MagicMock() 43 | logger.tqdm.set_description = MagicMock() 44 | logger.log = MagicMock() 45 | logger.set_tqdm_description("baz") 46 | logger.tqdm.set_description.assert_called_once_with("baz") 47 | logger.tqdm_write("qux") 48 | logger.tqdm.write.assert_called_once_with("qux") 49 | logger.log.assert_not_called 50 | -------------------------------------------------------------------------------- /tests/test_authentication.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | import pytest 4 | import os 5 | import click 6 | from click.testing import CliRunner 7 | from icloudpd.authentication import authenticate, TwoStepAuthRequiredError 8 | import pyicloud_ipd 9 | from icloudpd.base import main 10 | 11 | vcr = VCR(decode_compressed_response=True) 12 | 13 | 14 | class AuthenticationTestCase(TestCase): 15 | @pytest.fixture(autouse=True) 16 | def inject_fixtures(self, caplog): 17 | self._caplog = caplog 18 | 19 | def test_failed_auth(self): 20 | with vcr.use_cassette("tests/vcr_cassettes/failed_auth.yml"): 21 | with self.assertRaises( 22 | pyicloud_ipd.exceptions.PyiCloudFailedLoginException 23 | ) as context: 24 | authenticate( 25 | "bad_username", 26 | "bad_password", 27 | client_id="EC5646DE-9423-11E8-BF21-14109FE0B321", 28 | ) 29 | 30 | self.assertTrue("Invalid email/password combination." in str(context.exception)) 31 | 32 | def test_2sa_required(self): 33 | with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"): 34 | with self.assertRaises(TwoStepAuthRequiredError) as context: 35 | # To re-record this HTTP request, 36 | # delete ./tests/vcr_cassettes/auth_requires_2sa.yml, 37 | # put your actual credentials in here, run the test, 38 | # and then replace with dummy credentials. 39 | authenticate( 40 | "jdoe@gmail.com", 41 | "password1", 42 | raise_error_on_2sa=True, 43 | client_id="EC5646DE-9423-11E8-BF21-14109FE0B321", 44 | ) 45 | 46 | self.assertTrue( 47 | "Two-step/two-factor authentication is required!" 48 | in str(context.exception) 49 | ) 50 | 51 | def test_successful_auth(self): 52 | with vcr.use_cassette("tests/vcr_cassettes/successful_auth.yml"): 53 | authenticate( 54 | "jdoe@gmail.com", 55 | "password1", 56 | client_id="EC5646DE-9423-11E8-BF21-14109FE0B321", 57 | ) 58 | 59 | def test_password_prompt(self): 60 | if not os.path.exists("tests/fixtures/Photos"): 61 | os.makedirs("tests/fixtures/Photos") 62 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 63 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 64 | runner = CliRunner() 65 | result = runner.invoke( 66 | main, 67 | [ 68 | "--username", 69 | "jdoe@gmail.com", 70 | "--recent", 71 | "0", 72 | "--no-progress-bar", 73 | "tests/fixtures/Photos", 74 | ], 75 | input="password1\n", 76 | ) 77 | self.assertIn("DEBUG Authenticating...", self._caplog.text) 78 | self.assertIn( 79 | "DEBUG Looking up all photos and videos...", self._caplog.text 80 | ) 81 | self.assertIn( 82 | "INFO All photos have been downloaded!", self._caplog.text 83 | ) 84 | assert result.exit_code == 0 85 | -------------------------------------------------------------------------------- /icloudpd/download.py: -------------------------------------------------------------------------------- 1 | """Handles file downloads with retries and error handling""" 2 | 3 | import os 4 | import socket 5 | import time 6 | import logging 7 | from tzlocal import get_localzone 8 | from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin 9 | from pyicloud_ipd.exceptions import PyiCloudAPIResponseError 10 | from icloudpd.logger import setup_logger 11 | 12 | # Import the constants object so that we can mock WAIT_SECONDS in tests 13 | from icloudpd import constants 14 | 15 | 16 | def update_mtime(photo, download_path): 17 | """Set the modification time of the downloaded file to the photo creation date""" 18 | if photo.created: 19 | created_date = None 20 | try: 21 | created_date = photo.created.astimezone( 22 | get_localzone()) 23 | except (ValueError, OSError): 24 | # We already show the timezone conversion error in base.py, 25 | # when generating the download directory. 26 | # So just return silently without touching the mtime. 27 | return 28 | ctime = time.mktime(created_date.timetuple()) 29 | os.utime(download_path, (ctime, ctime)) 30 | 31 | 32 | def download_media(icloud, photo, download_path, size): 33 | """Download the photo to path, with retries and error handling""" 34 | logger = setup_logger() 35 | 36 | for retries in range(constants.MAX_RETRIES): 37 | try: 38 | photo_response = photo.download(size) 39 | if photo_response: 40 | with open(download_path, "wb") as file_obj: 41 | for chunk in photo_response.iter_content(chunk_size=1024): 42 | if chunk: 43 | file_obj.write(chunk) 44 | update_mtime(photo, download_path) 45 | return True 46 | 47 | logger.tqdm_write( 48 | "Could not find URL to download %s for size %s!" 49 | % (photo.filename, size), 50 | logging.ERROR, 51 | ) 52 | break 53 | 54 | except (ConnectionError, socket.timeout, PyiCloudAPIResponseError) as ex: 55 | if "Invalid global session" in str(ex): 56 | logger.tqdm_write( 57 | "Session error, re-authenticating...", 58 | logging.ERROR) 59 | if retries > 0: 60 | # If the first reauthentication attempt failed, 61 | # start waiting a few seconds before retrying in case 62 | # there are some issues with the Apple servers 63 | time.sleep(constants.WAIT_SECONDS) 64 | 65 | icloud.authenticate() 66 | else: 67 | logger.tqdm_write( 68 | "Error downloading %s, retrying after %d seconds..." 69 | % (photo.filename, constants.WAIT_SECONDS), 70 | logging.ERROR, 71 | ) 72 | time.sleep(constants.WAIT_SECONDS) 73 | 74 | except IOError: 75 | logger.error( 76 | "IOError while writing file to %s! " 77 | "You might have run out of disk space, or the file " 78 | "might be too large for your OS. " 79 | "Skipping this file...", download_path 80 | ) 81 | break 82 | else: 83 | logger.tqdm_write( 84 | "Could not download %s! Please try again later." % photo.filename 85 | ) 86 | 87 | return False 88 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from unittest import TestCase 3 | from vcr import VCR 4 | import os 5 | import click 6 | from click.testing import CliRunner 7 | from icloudpd.base import main 8 | 9 | vcr = VCR(decode_compressed_response=True) 10 | 11 | 12 | class CliTestCase(TestCase): 13 | def test_cli(self): 14 | runner = CliRunner() 15 | result = runner.invoke(main, ["--help"]) 16 | assert result.exit_code == 0 17 | 18 | def test_log_levels(self): 19 | if not os.path.exists("tests/fixtures/Photos"): 20 | os.makedirs("tests/fixtures/Photos") 21 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 22 | # Pass fixed client ID via environment variable 23 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 24 | runner = CliRunner() 25 | result = runner.invoke( 26 | main, 27 | [ 28 | "--username", 29 | "jdoe@gmail.com", 30 | "--password", 31 | "password1", 32 | "--recent", 33 | "0", 34 | "--log-level", 35 | "info", 36 | "tests/fixtures/Photos", 37 | ], 38 | ) 39 | assert result.exit_code == 0 40 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 41 | result = runner.invoke( 42 | main, 43 | [ 44 | "--username", 45 | "jdoe@gmail.com", 46 | "--password", 47 | "password1", 48 | "--recent", 49 | "0", 50 | "--log-level", 51 | "error", 52 | "tests/fixtures/Photos", 53 | ], 54 | ) 55 | assert result.exit_code == 0 56 | 57 | def test_tqdm(self): 58 | if not os.path.exists("tests/fixtures/Photos"): 59 | os.makedirs("tests/fixtures/Photos") 60 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 61 | # Force tqdm progress bar via ENV var 62 | os.environ["FORCE_TQDM"] = "yes" 63 | runner = CliRunner() 64 | result = runner.invoke( 65 | main, 66 | [ 67 | "--username", 68 | "jdoe@gmail.com", 69 | "--password", 70 | "password1", 71 | "--recent", 72 | "0", 73 | "tests/fixtures/Photos", 74 | ], 75 | ) 76 | del os.environ["FORCE_TQDM"] 77 | assert result.exit_code == 0 78 | 79 | def test_unicode_directory(self): 80 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 81 | # Pass fixed client ID via environment variable 82 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 83 | runner = CliRunner() 84 | result = runner.invoke( 85 | main, 86 | [ 87 | "--username", 88 | "jdoe@gmail.com", 89 | "--password", 90 | "password1", 91 | "--recent", 92 | "0", 93 | "--log-level", 94 | "info", 95 | "tests/fixtures/相片", 96 | ], 97 | ) 98 | assert result.exit_code == 0 99 | -------------------------------------------------------------------------------- /icloudpd/authentication.py: -------------------------------------------------------------------------------- 1 | """Handles username/password authentication and two-step authentication""" 2 | 3 | import sys 4 | import click 5 | import pyicloud_ipd 6 | from icloudpd.logger import setup_logger 7 | 8 | 9 | class TwoStepAuthRequiredError(Exception): 10 | """ 11 | Raised when 2SA is required. base.py catches this exception 12 | and sends an email notification. 13 | """ 14 | 15 | 16 | def authenticate( 17 | username, 18 | password, 19 | cookie_directory=None, 20 | raise_error_on_2sa=False, 21 | client_id=None 22 | ): 23 | """Authenticate with iCloud username and password""" 24 | logger = setup_logger() 25 | logger.debug("Authenticating...") 26 | try: 27 | # If password not provided on command line variable will be set to None 28 | # and PyiCloud will attempt to retrieve from it's keyring 29 | icloud = pyicloud_ipd.PyiCloudService( 30 | username, password, 31 | cookie_directory=cookie_directory, 32 | client_id=client_id) 33 | except pyicloud_ipd.exceptions.NoStoredPasswordAvailable: 34 | # Prompt for password if not stored in PyiCloud's keyring 35 | password = click.prompt("iCloud Password", hide_input=True) 36 | icloud = pyicloud_ipd.PyiCloudService( 37 | username, password, 38 | cookie_directory=cookie_directory, 39 | client_id=client_id) 40 | 41 | if icloud.requires_2sa: 42 | if raise_error_on_2sa: 43 | raise TwoStepAuthRequiredError( 44 | "Two-step/two-factor authentication is required!" 45 | ) 46 | logger.info("Two-step/two-factor authentication is required!") 47 | request_2sa(icloud, logger) 48 | return icloud 49 | 50 | 51 | def request_2sa(icloud, logger): 52 | """Request two-step authentication. Prompts for SMS or device""" 53 | devices = icloud.trusted_devices 54 | devices_count = len(devices) 55 | device_index = 0 56 | if devices_count > 0: 57 | for i, device in enumerate(devices): 58 | print( 59 | " %s: %s" % 60 | (i, device.get( 61 | "deviceName", "SMS to %s" % 62 | device.get("phoneNumber")))) 63 | 64 | # pylint: disable-msg=superfluous-parens 65 | print(" %s: Enter two-factor authentication code" % devices_count) 66 | # pylint: enable-msg=superfluous-parens 67 | device_index = click.prompt( 68 | "Please choose an option:", 69 | default=0, 70 | type=click.IntRange( 71 | 0, 72 | devices_count)) 73 | 74 | if device_index == devices_count: 75 | # We're using the 2FA code that was automatically sent to the user's device, 76 | # so can just use an empty dict() 77 | device = dict() 78 | else: 79 | device = devices[device_index] 80 | if not icloud.send_verification_code(device): 81 | logger.error("Failed to send two-factor authentication code") 82 | sys.exit(1) 83 | 84 | code = click.prompt("Please enter two-factor authentication code") 85 | if not icloud.validate_verification_code(device, code): 86 | logger.error("Failed to verify two-factor authentication code") 87 | sys.exit(1) 88 | logger.info( 89 | "Great, you're all set up. The script can now be run without " 90 | "user interaction until 2SA expires.\n" 91 | "You can set up email notifications for when " 92 | "the two-step authentication expires.\n" 93 | "(Use --help to view information about SMTP options.)" 94 | ) 95 | -------------------------------------------------------------------------------- /tests/test_autodelete_photos.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | import os 4 | import shutil 5 | import click 6 | import pytest 7 | import mock 8 | from click.testing import CliRunner 9 | import piexif 10 | from icloudpd.base import main 11 | import icloudpd.exif_datetime 12 | 13 | vcr = VCR(decode_compressed_response=True, record_mode="new_episodes") 14 | 15 | 16 | class AutodeletePhotosTestCase(TestCase): 17 | @pytest.fixture(autouse=True) 18 | def inject_fixtures(self, caplog): 19 | self._caplog = caplog 20 | 21 | def test_autodelete_photos(self): 22 | if os.path.exists("tests/fixtures/Photos"): 23 | shutil.rmtree("tests/fixtures/Photos") 24 | os.makedirs("tests/fixtures/Photos") 25 | 26 | # create some empty files that should be deleted 27 | os.makedirs("tests/fixtures/Photos/2018/07/30/") 28 | open("tests/fixtures/Photos/2018/07/30/IMG_7406.MOV", "a").close() 29 | os.makedirs("tests/fixtures/Photos/2018/07/26/") 30 | open("tests/fixtures/Photos/2018/07/26/IMG_7383.PNG", "a").close() 31 | os.makedirs("tests/fixtures/Photos/2018/07/12/") 32 | open("tests/fixtures/Photos/2018/07/12/IMG_7190.JPG", "a").close() 33 | open("tests/fixtures/Photos/2018/07/12/IMG_7190-medium.JPG", "a").close() 34 | 35 | # Should not be deleted 36 | open("tests/fixtures/Photos/2018/07/30/IMG_7407.JPG", "a").close() 37 | open("tests/fixtures/Photos/2018/07/30/IMG_7407-original.JPG", "a").close() 38 | 39 | with vcr.use_cassette("tests/vcr_cassettes/autodelete_photos.yml"): 40 | # Pass fixed client ID via environment variable 41 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 42 | runner = CliRunner() 43 | result = runner.invoke( 44 | main, 45 | [ 46 | "--username", 47 | "jdoe@gmail.com", 48 | "--password", 49 | "password1", 50 | "--recent", 51 | "0", 52 | "--skip-videos", 53 | "--auto-delete", 54 | "tests/fixtures/Photos", 55 | ], 56 | ) 57 | self.assertIn("DEBUG Looking up all photos...", self._caplog.text) 58 | self.assertIn( 59 | "INFO Downloading 0 original photos to tests/fixtures/Photos/ ...", 60 | self._caplog.text, 61 | ) 62 | self.assertIn( 63 | "INFO All photos have been downloaded!", self._caplog.text 64 | ) 65 | self.assertIn( 66 | "INFO Deleting any files found in 'Recently Deleted'...", 67 | self._caplog.text, 68 | ) 69 | 70 | self.assertIn( 71 | "INFO Deleting any files found in 'Recently Deleted'...", 72 | self._caplog.text, 73 | ) 74 | 75 | self.assertIn( 76 | "INFO Deleting tests/fixtures/Photos/2018/07/30/IMG_7406.MOV", 77 | self._caplog.text, 78 | ) 79 | self.assertIn( 80 | "INFO Deleting tests/fixtures/Photos/2018/07/26/IMG_7383.PNG", 81 | self._caplog.text, 82 | ) 83 | self.assertIn( 84 | "INFO Deleting tests/fixtures/Photos/2018/07/12/IMG_7190.JPG", 85 | self._caplog.text, 86 | ) 87 | self.assertIn( 88 | "INFO Deleting tests/fixtures/Photos/2018/07/12/IMG_7190-medium.JPG", 89 | self._caplog.text, 90 | ) 91 | 92 | self.assertNotIn("IMG_7407.JPG", self._caplog.text) 93 | self.assertNotIn("IMG_7407-original.JPG", self._caplog.text) 94 | 95 | assert result.exit_code == 0 96 | -------------------------------------------------------------------------------- /tests/test_email_notifications.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | from mock import patch 4 | from freezegun import freeze_time 5 | import smtplib 6 | import os 7 | import click 8 | from click.testing import CliRunner 9 | from icloudpd.base import main 10 | 11 | import pyicloud_ipd 12 | 13 | vcr = VCR(decode_compressed_response=True) 14 | 15 | 16 | class EmailNotificationsTestCase(TestCase): 17 | @freeze_time("2018-01-01") 18 | def test_2sa_required_email_notification(self): 19 | with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"): 20 | with patch("smtplib.SMTP") as smtp: 21 | # Pass fixed client ID via environment variable 22 | os.environ["CLIENT_ID"] = "EC5646DE-9423-11E8-BF21-14109FE0B321" 23 | runner = CliRunner() 24 | result = runner.invoke( 25 | main, 26 | [ 27 | "--username", 28 | "jdoe@gmail.com", 29 | "--password", 30 | "password1", 31 | "--smtp-username", 32 | "jdoe+smtp@gmail.com", 33 | "--smtp-password", 34 | "password1", 35 | "--notification-email", 36 | "jdoe+notifications@gmail.com", 37 | "tests/fixtures/Photos", 38 | ], 39 | ) 40 | print(result.output) 41 | assert result.exit_code == 1 42 | smtp_instance = smtp() 43 | smtp_instance.connect.assert_called_once() 44 | smtp_instance.starttls.assert_called_once() 45 | smtp_instance.login.assert_called_once_with( 46 | "jdoe+smtp@gmail.com", "password1" 47 | ) 48 | smtp_instance.sendmail.assert_called_once_with( 49 | "jdoe+smtp@gmail.com", 50 | "jdoe+notifications@gmail.com", 51 | "From: iCloud Photos Downloader \n" 52 | "To: jdoe+notifications@gmail.com\n" 53 | "Subject: icloud_photos_downloader: Two step authentication has expired\n" 54 | "Date: 01/01/2018 00:00\n\nHello,\n\n" 55 | "Two-step authentication has expired for the icloud_photos_downloader script.\n" 56 | "Please log in to your server and run the script manually to update two-step " 57 | "authentication.", 58 | ) 59 | 60 | @freeze_time("2018-01-01") 61 | def test_2sa_notification_without_smtp_login_and_tls(self): 62 | with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"): 63 | with patch("smtplib.SMTP") as smtp: 64 | # Pass fixed client ID via environment variable 65 | os.environ["CLIENT_ID"] = "EC5646DE-9423-11E8-BF21-14109FE0B321" 66 | runner = CliRunner() 67 | result = runner.invoke( 68 | main, 69 | [ 70 | "--username", 71 | "jdoe@gmail.com", 72 | "--password", 73 | "password1", 74 | "--smtp-no-tls", 75 | "--notification-email", 76 | "jdoe+notifications@gmail.com", 77 | "tests/fixtures/Photos", 78 | ], 79 | ) 80 | print(result.output) 81 | assert result.exit_code == 1 82 | smtp_instance = smtp() 83 | smtp_instance.connect.assert_called_once() 84 | smtp_instance.starttls.assert_not_called() 85 | smtp_instance.login.assert_not_called() 86 | smtp_instance.sendmail.assert_called_once_with( 87 | "jdoe+notifications@gmail.com", 88 | "jdoe+notifications@gmail.com", 89 | "From: iCloud Photos Downloader \n" 90 | "To: jdoe+notifications@gmail.com\n" 91 | "Subject: icloud_photos_downloader: Two step authentication has expired\n" 92 | "Date: 01/01/2018 00:00\n\nHello,\n\n" 93 | "Two-step authentication has expired for the icloud_photos_downloader script.\n" 94 | "Please log in to your server and run the script manually to update two-step " 95 | "authentication.", 96 | ) 97 | 98 | @freeze_time("2018-01-01") 99 | def test_2sa_required_notification_script(self): 100 | with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"): 101 | with patch("subprocess.call") as subprocess_patched: 102 | # Pass fixed client ID via environment variable 103 | os.environ["CLIENT_ID"] = "EC5646DE-9423-11E8-BF21-14109FE0B321" 104 | runner = CliRunner() 105 | result = runner.invoke( 106 | main, 107 | [ 108 | "--username", 109 | "jdoe@gmail.com", 110 | "--password", 111 | "password1", 112 | "--notification-script", 113 | "./test_script.sh", 114 | "tests/fixtures/Photos", 115 | ], 116 | ) 117 | print(result.output) 118 | assert result.exit_code == 1 119 | subprocess_patched.assert_called_once_with(["./test_script.sh"]) 120 | -------------------------------------------------------------------------------- /tests/vcr_cassettes/auth_requires_2sa.yml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: !!python/unicode '{"apple_id": "jdoe@gmail.com", "password": "password1", 4 | "extended_login": false}' 5 | headers: 6 | Accept: ['*/*'] 7 | Accept-Encoding: ['gzip, deflate'] 8 | Connection: [keep-alive] 9 | Content-Length: ['88'] 10 | Origin: ['https://www.icloud.com'] 11 | Referer: ['https://www.icloud.com/'] 12 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 13 | method: POST 14 | uri: https://setup.icloud.com/setup/ws/1/login?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&clientId=EC5646DE-9423-11E8-BF21-14109FE0B321&ckjsVersion=2.0.5&ckjsBuildVersion=17DProjectDev77 15 | response: 16 | body: {string: !!python/unicode '{"dsInfo":{"lastName":"Doe","iCDPEnabled":false,"dsid":"123456789","hsaEnabled":true,"ironcadeMigrated":true,"locale":"en-us_US","brZoneConsolidated":false,"isManagedAppleID":false,"gilligan-invited":"true","appleIdAliases":["jdoe@icloud.com"],"hsaVersion":2,"isPaidDeveloper":true,"countryCode":"USA","notificationId":"12341234-1234-1234-1234-143241234123","primaryEmailVerified":true,"aDsID":"12341234123412341234","locked":false,"hasICloudQualifyingDevice":true,"primaryEmail":"jdoe@gmail.com","appleIdEntries":[{"isPrimary":true,"type":"EMAIL","value":"jdoe@gmail.com"}],"gilligan-enabled":"true","fullName":"John 17 | Doe","languageCode":"en-us","appleId":"jdoe@gmail.com","firstName":"John","iCloudAppleIdAlias":"jdoe@icloud.com","notesMigrated":true,"hasPaymentInfo":true,"pcsDeleted":false,"appleIdAlias":"","brMigrated":true,"statusCode":2},"hasMinimumDeviceForPhotosWeb":true,"iCDPEnabled":false,"webservices":{"reminders":{"url":"https://p10-remindersws.icloud.com:443","status":"active"},"notes":{"url":"https://p10-notesws.icloud.com:443","status":"active"},"mail":{"url":"https://p10-mailws.icloud.com:443","status":"active"},"ckdatabasews":{"pcsRequired":true,"url":"https://p10-ckdatabasews.icloud.com:443","status":"active"},"photosupload":{"pcsRequired":true,"url":"https://p10-uploadphotosws.icloud.com:443","status":"active"},"photos":{"pcsRequired":true,"uploadUrl":"https://p10-uploadphotosws.icloud.com:443","url":"https://p10-photosws.icloud.com:443","status":"active"},"drivews":{"pcsRequired":true,"url":"https://p10-drivews.icloud.com:443","status":"active"},"uploadimagews":{"url":"https://p10-uploadimagews.icloud.com:443","status":"active"},"schoolwork":{},"cksharews":{"url":"https://p10-ckshare.icloud.com:443","status":"active"},"findme":{"url":"https://p10-fmipweb.icloud.com:443","status":"active"},"ckdeviceservice":{"url":"https://p10-ckdevice.icloud.com:443"},"iworkthumbnailws":{"url":"https://p10-iworkthumbnailws.icloud.com:443","status":"active"},"calendar":{"url":"https://p10-calendarws.icloud.com:443","status":"active"},"docws":{"pcsRequired":true,"url":"https://p10-docws.icloud.com:443","status":"active"},"settings":{"url":"https://p10-settingsws.icloud.com:443","status":"active"},"ubiquity":{"url":"https://p10-ubiquityws.icloud.com:443","status":"active"},"streams":{"url":"https://p10-streams.icloud.com:443","status":"active"},"keyvalue":{"url":"https://p10-keyvalueservice.icloud.com:443","status":"active"},"archivews":{"url":"https://p10-archivews.icloud.com:443","status":"active"},"push":{"url":"https://p10-pushws.icloud.com:443","status":"active"},"iwmb":{"url":"https://p10-iwmb.icloud.com:443","status":"active"},"iworkexportws":{"url":"https://p10-iworkexportws.icloud.com:443","status":"active"},"geows":{"url":"https://p10-geows.icloud.com:443","status":"active"},"account":{"iCloudEnv":{"shortId":"p","vipSuffix":"p"},"url":"https://p10-setup.icloud.com:443","status":"active"},"fmf":{"url":"https://p10-fmfweb.icloud.com:443","status":"active"},"contacts":{"url":"https://p10-contactsws.icloud.com:443","status":"active"}},"pcsEnabled":true,"configBag":{"urls":{"accountCreateUI":"https://appleid.apple.com/widget/account/?widgetKey=12312412412341234123412341234123412341234#!create","accountLoginUI":"https://idmsa.apple.com/appleauth/auth/signin?widgetKey=83545bf919730e51dbfba24e7e8a78d2","accountLogin":"https://setup.icloud.com/setup/ws/1/accountLogin","accountRepairUI":"https://appleid.apple.com/widget/account/?widgetKey=12312412412341234123412341234123412341234#!repair","downloadICloudTerms":"https://setup.icloud.com/setup/ws/1/downloadLiteTerms","repairDone":"https://setup.icloud.com/setup/ws/1/repairDone","vettingUrlForEmail":"https://id.apple.com/IDMSEmailVetting/vetShareEmail","accountCreate":"https://setup.icloud.com/setup/ws/1/createLiteAccount","getICloudTerms":"https://setup.icloud.com/setup/ws/1/getTerms","vettingUrlForPhone":"https://id.apple.com/IDMSEmailVetting/vetSharePhone"},"accountCreateEnabled":"true"},"hsaTrustedBrowser":false,"appsOrder":["mail","contacts","calendar","photos","iclouddrive","notes2","reminders","pages","numbers","keynote","newspublisher","fmf","find","settings"],"version":2,"isExtendedLogin":false,"pcsServiceIdentitiesIncluded":false,"hsaChallengeRequired":true,"requestInfo":{"country":"TH","timeZone":"GMT+7","isAppleInternal":true},"pcsDeleted":false,"iCloudInfo":{"SafariBookmarksHasMigratedToCloudKit":false},"apps":{"calendar":{},"reminders":{},"keynote":{"isQualifiedForBeta":true},"settings":{"canLaunchWithOneFactor":true},"mail":{},"numbers":{"isQualifiedForBeta":true},"photos":{},"pages":{"isQualifiedForBeta":true},"find":{"canLaunchWithOneFactor":true},"notes2":{},"iclouddrive":{},"newspublisher":{"isHidden":true},"fmf":{},"contacts":{}}}'} 18 | headers: 19 | access-control-allow-credentials: ['true'] 20 | access-control-allow-origin: ['https://www.icloud.com'] 21 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 22 | apple-originating-system: [UnknownOriginatingSystem] 23 | apple-seq: ['0'] 24 | apple-tk: ['false'] 25 | cache-control: ['no-cache, no-store, private'] 26 | connection: [keep-alive] 27 | content-length: ['4895'] 28 | content-type: [application/json; charset=UTF-8] 29 | date: ['Mon, 30 Jul 2018 19:00:39 GMT'] 30 | server: [AppleHttpServer/2f080fc0] 31 | strict-transport-security: [max-age=31536000; includeSubDomains] 32 | via: ['icloudedge:si03p00ic-ztde010417:7401:18RC341:Singapore'] 33 | x-apple-jingle-correlation-key: [SJHIUN7879234KJHH8JBH] 34 | x-apple-request-uuid: [NISUHFIOSUHFOSIDUHFOSIDF] 35 | x-responding-instance: ['setupservice:328457238759328579234875'] 36 | status: {code: 200, message: OK} 37 | version: 1 38 | -------------------------------------------------------------------------------- /tests/vcr_cassettes/successful_auth.yml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: !!python/unicode '{"apple_id": "jdoe@gmail.com", "password": "password1", 4 | "extended_login": false}' 5 | headers: 6 | Accept: ['*/*'] 7 | Accept-Encoding: ['gzip, deflate'] 8 | Connection: [keep-alive] 9 | Content-Length: ['88'] 10 | Origin: ['https://www.icloud.com'] 11 | Referer: ['https://www.icloud.com/'] 12 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 13 | method: POST 14 | uri: https://setup.icloud.com/setup/ws/1/login?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&clientId=EC5646DE-9423-11E8-BF21-14109FE0B321&ckjsVersion=2.0.5&ckjsBuildVersion=17DProjectDev77 15 | response: 16 | body: {string: !!python/unicode '{"dsInfo":{"lastName":"Doe","iCDPEnabled":false,"dsid":"123456789","hsaEnabled":true,"ironcadeMigrated":true,"locale":"en-us_US","brZoneConsolidated":false,"isManagedAppleID":false,"gilligan-invited":"true","appleIdAliases":["jdoe@icloud.com"],"hsaVersion":2,"isPaidDeveloper":true,"countryCode":"USA","notificationId":"234234234c95-863a-234234b4","primaryEmailVerified":true,"aDsID":"23423423423432c2-d234234680","locked":false,"hasICloudQualifyingDevice":true,"primaryEmail":"jdoe@gmail.com","appleIdEntries":[{"isPrimary":true,"type":"EMAIL","value":"jdoe@gmail.com"}],"gilligan-enabled":"true","fullName":"John 17 | Doe","languageCode":"en-us","appleId":"jdoe@gmail.com","firstName":"John","iCloudAppleIdAlias":"jdoe@icloud.com","notesMigrated":true,"hasPaymentInfo":false,"pcsDeleted":false,"appleIdAlias":"","brMigrated":true,"statusCode":2},"hasMinimumDeviceForPhotosWeb":true,"iCDPEnabled":false,"webservices":{"reminders":{"url":"https://p10-remindersws.icloud.com:443","status":"active"},"notes":{"url":"https://p10-notesws.icloud.com:443","status":"active"},"mail":{"url":"https://p10-mailws.icloud.com:443","status":"active"},"ckdatabasews":{"pcsRequired":true,"url":"https://p10-ckdatabasews.icloud.com:443","status":"active"},"photosupload":{"pcsRequired":true,"url":"https://p10-uploadphotosws.icloud.com:443","status":"active"},"photos":{"pcsRequired":true,"uploadUrl":"https://p10-uploadphotosws.icloud.com:443","url":"https://p10-photosws.icloud.com:443","status":"active"},"drivews":{"pcsRequired":true,"url":"https://p10-drivews.icloud.com:443","status":"active"},"uploadimagews":{"url":"https://p10-uploadimagews.icloud.com:443","status":"active"},"schoolwork":{},"cksharews":{"url":"https://p10-ckshare.icloud.com:443","status":"active"},"findme":{"url":"https://p10-fmipweb.icloud.com:443","status":"active"},"ckdeviceservice":{"url":"https://p10-ckdevice.icloud.com:443"},"iworkthumbnailws":{"url":"https://p10-iworkthumbnailws.icloud.com:443","status":"active"},"calendar":{"url":"https://p10-calendarws.icloud.com:443","status":"active"},"docws":{"pcsRequired":true,"url":"https://p10-docws.icloud.com:443","status":"active"},"settings":{"url":"https://p10-settingsws.icloud.com:443","status":"active"},"ubiquity":{"url":"https://p10-ubiquityws.icloud.com:443","status":"active"},"streams":{"url":"https://p10-streams.icloud.com:443","status":"active"},"keyvalue":{"url":"https://p10-keyvalueservice.icloud.com:443","status":"active"},"archivews":{"url":"https://p10-archivews.icloud.com:443","status":"active"},"push":{"url":"https://p10-pushws.icloud.com:443","status":"active"},"iwmb":{"url":"https://p10-iwmb.icloud.com:443","status":"active"},"iworkexportws":{"url":"https://p10-iworkexportws.icloud.com:443","status":"active"},"geows":{"url":"https://p10-geows.icloud.com:443","status":"active"},"account":{"iCloudEnv":{"shortId":"p","vipSuffix":"p"},"url":"https://p10-setup.icloud.com:443","status":"active"},"fmf":{"url":"https://p10-fmfweb.icloud.com:443","status":"active"},"contacts":{"url":"https://p10-contactsws.icloud.com:443","status":"active"}},"pcsEnabled":true,"configBag":{"urls":{"accountCreateUI":"https://appleid.apple.com/widget/account/?widgetKey=d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d#!create","accountLoginUI":"https://idmsa.apple.com/appleauth/auth/signin?widgetKey=83545bf919730e51dbfba24e7e8a78d2","accountLogin":"https://setup.icloud.com/setup/ws/1/accountLogin","accountRepairUI":"https://appleid.apple.com/widget/account/?widgetKey=d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d#!repair","downloadICloudTerms":"https://setup.icloud.com/setup/ws/1/downloadLiteTerms","repairDone":"https://setup.icloud.com/setup/ws/1/repairDone","vettingUrlForEmail":"https://id.apple.com/IDMSEmailVetting/vetShareEmail","accountCreate":"https://setup.icloud.com/setup/ws/1/createLiteAccount","getICloudTerms":"https://setup.icloud.com/setup/ws/1/getTerms","vettingUrlForPhone":"https://id.apple.com/IDMSEmailVetting/vetSharePhone"},"accountCreateEnabled":"true"},"hsaTrustedBrowser":true,"appsOrder":["mail","contacts","calendar","photos","iclouddrive","notes2","reminders","pages","numbers","keynote","newspublisher","fmf","find","settings"],"version":2,"isExtendedLogin":false,"pcsServiceIdentitiesIncluded":true,"hsaChallengeRequired":false,"requestInfo":{"country":"TH","timeZone":"GMT+7","isAppleInternal":true},"pcsDeleted":false,"iCloudInfo":{"SafariBookmarksHasMigratedToCloudKit":false},"apps":{"calendar":{},"reminders":{},"keynote":{"isQualifiedForBeta":true},"settings":{"canLaunchWithOneFactor":true},"mail":{},"numbers":{"isQualifiedForBeta":true},"photos":{},"pages":{"isQualifiedForBeta":true},"find":{"canLaunchWithOneFactor":true},"notes2":{},"iclouddrive":{},"newspublisher":{"isHidden":true},"fmf":{},"contacts":{}}}'} 18 | headers: 19 | access-control-allow-credentials: ['true'] 20 | access-control-allow-origin: ['https://www.icloud.com'] 21 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 22 | apple-originating-system: [UnknownOriginatingSystem] 23 | apple-seq: ['0'] 24 | apple-tk: ['false'] 25 | cache-control: ['no-cache, no-store, private'] 26 | connection: [keep-alive] 27 | content-length: ['4895'] 28 | content-type: [application/json; charset=UTF-8] 29 | date: ['Mon, 30 Jul 2018 19:02:14 GMT'] 30 | server: [AppleHttpServer/2f080fc0] 31 | strict-transport-security: [max-age=31536000; includeSubDomains] 32 | via: ['icloudedge:si03p01ic-ztde010422:7401:18RC341:Singapore'] 33 | x-apple-jingle-correlation-key: [SDFUHISDUFHIDSUFHISDUHFDF] 34 | x-apple-request-uuid: [SDFOISDUMFIUSDMFIUHDSIFUHDSJF] 35 | x-responding-instance: ['setupservice:12341234234:mr90p54ic-12341234234:8003:1813B80:3b85e7d76'] 36 | status: {code: 200, message: OK} 37 | version: 1 38 | -------------------------------------------------------------------------------- /tests/test_two_step_auth.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | import mock 4 | import pytest 5 | import os 6 | import click 7 | from click.testing import CliRunner 8 | from icloudpd.base import main 9 | from pyicloud_ipd import PyiCloudService 10 | 11 | vcr = VCR(decode_compressed_response=True) 12 | 13 | 14 | class TwoStepAuthTestCase(TestCase): 15 | @pytest.fixture(autouse=True) 16 | def inject_fixtures(self, caplog): 17 | self._caplog = caplog 18 | 19 | def test_2sa_flow_invalid_device_2fa(self): 20 | with vcr.use_cassette("tests/vcr_cassettes/2sa_flow_invalid_device.yml"): 21 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 22 | runner = CliRunner() 23 | result = runner.invoke( 24 | main, 25 | [ 26 | "--username", 27 | "jdoe@gmail.com", 28 | "--password", 29 | "password1", 30 | "--recent", 31 | "0", 32 | "--no-progress-bar", 33 | "tests/fixtures/Photos", 34 | ], 35 | input="1\n901431\n", 36 | ) 37 | self.assertIn( 38 | "ERROR Failed to verify two-factor authentication code", 39 | self._caplog.text, 40 | ) 41 | assert result.exit_code == 1 42 | 43 | def test_2sa_flow_device_2fa(self): 44 | with vcr.use_cassette("tests/vcr_cassettes/2sa_flow_valid_device.yml"): 45 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 46 | runner = CliRunner() 47 | result = runner.invoke( 48 | main, 49 | [ 50 | "--username", 51 | "jdoe@gmail.com", 52 | "--password", 53 | "password1", 54 | "--recent", 55 | "0", 56 | "--no-progress-bar", 57 | "tests/fixtures/Photos", 58 | ], 59 | input="1\n654321\n", 60 | ) 61 | self.assertIn("DEBUG Authenticating...", self._caplog.text) 62 | self.assertIn( 63 | "INFO Two-step/two-factor authentication is required!", 64 | self._caplog.text, 65 | ) 66 | self.assertIn(" 0: SMS to *******03", result.output) 67 | self.assertIn(" 1: Enter two-factor authentication code", result.output) 68 | self.assertIn("Please choose an option: [0]: 1", result.output) 69 | self.assertIn( 70 | "Please enter two-factor authentication code: 654321", result.output 71 | ) 72 | self.assertIn( 73 | "INFO Great, you're all set up. The script can now be run without " 74 | "user interaction until 2SA expires.", 75 | self._caplog.text, 76 | ) 77 | self.assertIn( 78 | "DEBUG Looking up all photos and videos...", self._caplog.text 79 | ) 80 | self.assertIn( 81 | "INFO All photos have been downloaded!", self._caplog.text 82 | ) 83 | assert result.exit_code == 0 84 | 85 | def test_2sa_flow_sms(self): 86 | with vcr.use_cassette("tests/vcr_cassettes/2sa_flow_valid_sms.yml"): 87 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 88 | runner = CliRunner() 89 | result = runner.invoke( 90 | main, 91 | [ 92 | "--username", 93 | "jdoe@gmail.com", 94 | "--password", 95 | "password1", 96 | "--recent", 97 | "0", 98 | "--no-progress-bar", 99 | "tests/fixtures/Photos", 100 | ], 101 | input="0\n123456\n", 102 | ) 103 | self.assertIn("DEBUG Authenticating...", self._caplog.text) 104 | self.assertIn( 105 | "INFO Two-step/two-factor authentication is required!", 106 | self._caplog.text, 107 | ) 108 | self.assertIn(" 0: SMS to *******03", result.output) 109 | self.assertIn(" 1: Enter two-factor authentication code", result.output) 110 | self.assertIn("Please choose an option: [0]: 0", result.output) 111 | self.assertIn( 112 | "Please enter two-factor authentication code: 123456", result.output 113 | ) 114 | self.assertIn( 115 | "INFO Great, you're all set up. The script can now be run without " 116 | "user interaction until 2SA expires.", 117 | self._caplog.text, 118 | ) 119 | self.assertIn( 120 | "DEBUG Looking up all photos and videos...", self._caplog.text 121 | ) 122 | self.assertIn( 123 | "INFO All photos have been downloaded!", self._caplog.text 124 | ) 125 | assert result.exit_code == 0 126 | 127 | def test_2sa_flow_sms_failed(self): 128 | with vcr.use_cassette("tests/vcr_cassettes/2sa_flow_valid_sms.yml"): 129 | with mock.patch.object( 130 | PyiCloudService, "send_verification_code" 131 | ) as svc_mocked: 132 | svc_mocked.return_value = False 133 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 134 | runner = CliRunner() 135 | result = runner.invoke( 136 | main, 137 | [ 138 | "--username", 139 | "jdoe@gmail.com", 140 | "--password", 141 | "password1", 142 | "--recent", 143 | "0", 144 | "--no-progress-bar", 145 | "tests/fixtures/Photos", 146 | ], 147 | input="0\n", 148 | ) 149 | self.assertIn("DEBUG Authenticating...", self._caplog.text) 150 | self.assertIn( 151 | "INFO Two-step/two-factor authentication is required!", 152 | self._caplog.text, 153 | ) 154 | self.assertIn(" 0: SMS to *******03", result.output) 155 | self.assertIn( 156 | " 1: Enter two-factor authentication code", result.output 157 | ) 158 | self.assertIn("Please choose an option: [0]: 0", result.output) 159 | self.assertIn( 160 | "ERROR Failed to send two-factor authentication code", 161 | self._caplog.text, 162 | ) 163 | assert result.exit_code == 1 164 | -------------------------------------------------------------------------------- /tests/test_listing_recent_photos.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | import os 4 | import shutil 5 | import click 6 | from click.testing import CliRunner 7 | import json 8 | import mock 9 | from icloudpd.base import main 10 | from tests.helpers.print_result_exception import print_result_exception 11 | 12 | vcr = VCR(decode_compressed_response=True) 13 | 14 | class ListingRecentPhotosTestCase(TestCase): 15 | def test_listing_recent_photos(self): 16 | if os.path.exists("tests/fixtures/Photos"): 17 | shutil.rmtree("tests/fixtures/Photos") 18 | os.makedirs("tests/fixtures/Photos") 19 | 20 | # Note - This test uses the same cassette as test_download_photos.py 21 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 22 | # Pass fixed client ID via environment variable 23 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 24 | runner = CliRunner() 25 | result = runner.invoke( 26 | main, 27 | [ 28 | "--username", 29 | "jdoe@gmail.com", 30 | "--password", 31 | "password1", 32 | "--recent", 33 | "5", 34 | "--only-print-filenames", 35 | "--no-progress-bar", 36 | "tests/fixtures/Photos", 37 | ], 38 | ) 39 | print_result_exception(result) 40 | filenames = result.output.splitlines() 41 | 42 | self.assertEqual(len(filenames), 8) 43 | self.assertEqual( 44 | "tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", filenames[0] 45 | ) 46 | self.assertEqual( 47 | "tests/fixtures/Photos/2018/07/31/IMG_7409.MOV", filenames[1] 48 | ) 49 | self.assertEqual( 50 | "tests/fixtures/Photos/2018/07/30/IMG_7408.JPG", filenames[2] 51 | ) 52 | self.assertEqual( 53 | "tests/fixtures/Photos/2018/07/30/IMG_7408.MOV", filenames[3] 54 | ) 55 | self.assertEqual( 56 | "tests/fixtures/Photos/2018/07/30/IMG_7407.JPG", filenames[4] 57 | ) 58 | self.assertEqual( 59 | "tests/fixtures/Photos/2018/07/30/IMG_7407.MOV", filenames[5] 60 | ) 61 | self.assertEqual( 62 | "tests/fixtures/Photos/2018/07/30/IMG_7405.MOV", filenames[6] 63 | ) 64 | self.assertEqual( 65 | "tests/fixtures/Photos/2018/07/30/IMG_7404.MOV", filenames[7] 66 | ) 67 | 68 | 69 | assert result.exit_code == 0 70 | 71 | def test_listing_recent_photos_with_missing_filenameEnc(self): 72 | if os.path.exists("tests/fixtures/Photos"): 73 | shutil.rmtree("tests/fixtures/Photos") 74 | os.makedirs("tests/fixtures/Photos") 75 | 76 | # Note - This test uses the same cassette as test_download_photos.py 77 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos_missing_filenameEnc.yml"): 78 | with mock.patch("icloudpd.base.open", create=True) as mock_open: 79 | with mock.patch.object(json, "dump") as mock_json: 80 | # Pass fixed client ID via environment variable 81 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 82 | runner = CliRunner() 83 | result = runner.invoke( 84 | main, 85 | [ 86 | "--username", 87 | "jdoe@gmail.com", 88 | "--password", 89 | "password1", 90 | "--recent", 91 | "5", 92 | "--only-print-filenames", 93 | "--no-progress-bar", 94 | "tests/fixtures/Photos", 95 | ], 96 | ) 97 | print_result_exception(result) 98 | 99 | self.assertEqual.__self__.maxDiff = None 100 | 101 | filenames = result.output.splitlines() 102 | 103 | # self.assertEqual(len(filenames), 5) 104 | self.assertEqual( 105 | "tests/fixtures/Photos/2018/07/31/AY6c_BsE0jja.JPG", filenames[0] 106 | ) 107 | self.assertEqual( 108 | "tests/fixtures/Photos/2018/07/31/AY6c_BsE0jja.MOV", filenames[1] 109 | ) 110 | self.assertEqual( 111 | "tests/fixtures/Photos/2018/07/30/IMG_7408.JPG", filenames[2] 112 | ) 113 | self.assertEqual( 114 | "tests/fixtures/Photos/2018/07/30/IMG_7408.MOV", filenames[3] 115 | ) 116 | self.assertEqual( 117 | "tests/fixtures/Photos/2018/07/30/AZ_wAGT9P6jh.JPG", filenames[4] 118 | ) 119 | assert result.exit_code == 0 120 | 121 | 122 | # This was used to solve the missing filenameEnc error. I found 123 | # another case where it might crash. (Maybe Apple changes the downloadURL key) 124 | def test_listing_recent_photos_with_missing_downloadURL(self): 125 | if os.path.exists("tests/fixtures/Photos"): 126 | shutil.rmtree("tests/fixtures/Photos") 127 | os.makedirs("tests/fixtures/Photos") 128 | 129 | # Note - This test uses the same cassette as test_download_photos.py 130 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos_missing_downloadUrl.yml"): 131 | with mock.patch("icloudpd.base.open", create=True) as mock_open: 132 | with mock.patch.object(json, "dump") as mock_json: 133 | # Pass fixed client ID via environment variable 134 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 135 | runner = CliRunner() 136 | result = runner.invoke( 137 | main, 138 | [ 139 | "--username", 140 | "jdoe@gmail.com", 141 | "--password", 142 | "password1", 143 | "--recent", 144 | "1", 145 | "--only-print-filenames", 146 | "--no-progress-bar", 147 | "tests/fixtures/Photos", 148 | ], 149 | ) 150 | print_result_exception(result) 151 | 152 | self.assertEqual.__self__.maxDiff = None 153 | self.assertEqual("""\ 154 | KeyError: 'downloadURL' attribute was not found in the photo fields! 155 | icloudpd has saved the photo record to: ./icloudpd-photo-error.json 156 | Please create a Gist with the contents of this file: https://gist.github.com 157 | Then create an issue on GitHub: https://github.com/ndbroadbent/icloud_photos_downloader/issues 158 | Include a link to the Gist in your issue, so that we can see what went wrong. 159 | 160 | """ , result.output) 161 | mock_open.assert_called_once_with('icloudpd-photo-error.json', 'w') 162 | mock_json.assert_called_once() 163 | # Check a few keys in the dict 164 | first_arg = mock_json.call_args_list[0][0][0] 165 | self.assertEqual( 166 | first_arg['master_record']['recordName'], 167 | 'AY6c+BsE0jjaXx9tmVGJM1D2VcEO') 168 | self.assertEqual( 169 | first_arg['master_record']['fields']['resVidSmallHeight']['value'], 170 | 581) 171 | self.assertEqual( 172 | first_arg['asset_record']['recordName'], 173 | 'F2A23C38-0020-42FE-A273-2923ADE3CAED') 174 | self.assertEqual( 175 | first_arg['asset_record']['fields']['assetDate']['value'], 176 | 1533021744816) 177 | assert result.exit_code == 0 178 | 179 | -------------------------------------------------------------------------------- /tests/vcr_cassettes/2sa_flow_invalid_device.yml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"apple_id": "jdoe@gmail.com", "password": "password1", "extended_login": 4 | false}' 5 | headers: 6 | Accept: ['*/*'] 7 | Accept-Encoding: ['gzip, deflate'] 8 | Connection: [keep-alive] 9 | Content-Length: ['88'] 10 | Origin: ['https://www.icloud.com'] 11 | Referer: ['https://www.icloud.com/'] 12 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 13 | method: POST 14 | uri: https://setup.icloud.com/setup/ws/1/login?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&ckjsBuildVersion=17DProjectDev77&ckjsVersion=2.0.5&clientId=DE309E26-942E-11E8-92F5-14109FE0B321 15 | response: 16 | body: {string: '{"dsInfo":{"lastName":"Doe","iCDPEnabled":false,"dsid":"185776146","hsaEnabled":true,"ironcadeMigrated":true,"locale":"en-us_US","brZoneConsolidated":false,"isManagedAppleID":false,"gilligan-invited":"true","appleIdAliases":["jdoe@icloud.com"],"hsaVersion":2,"isPaidDeveloper":true,"countryCode":"USA","notificationId":"ab6bfad3-e110-414f-9be0-125eff24efae","primaryEmailVerified":true,"aDsID":"001640-05-8c09a45f-3769-4206-a0c2-d88f3b815680","locked":false,"hasICloudQualifyingDevice":true,"primaryEmail":"jdoe@gmail.com","appleIdEntries":[{"isPrimary":true,"type":"EMAIL","value":"jdoe@gmail.com"}],"gilligan-enabled":"true","fullName":"John 17 | Doe","languageCode":"en-us","appleId":"jdoe@gmail.com","firstName":"John","iCloudAppleIdAlias":"jdoe@icloud.com","notesMigrated":true,"hasPaymentInfo":true,"pcsDeleted":false,"appleIdAlias":"","brMigrated":true,"statusCode":2},"hasMinimumDeviceForPhotosWeb":true,"iCDPEnabled":false,"webservices":{"reminders":{"url":"https://p10-remindersws.icloud.com:443","status":"active"},"notes":{"url":"https://p10-notesws.icloud.com:443","status":"active"},"mail":{"url":"https://p10-mailws.icloud.com:443","status":"active"},"ckdatabasews":{"pcsRequired":true,"url":"https://p10-ckdatabasews.icloud.com:443","status":"active"},"photosupload":{"pcsRequired":true,"url":"https://p10-uploadphotosws.icloud.com:443","status":"active"},"photos":{"pcsRequired":true,"uploadUrl":"https://p10-uploadphotosws.icloud.com:443","url":"https://p10-photosws.icloud.com:443","status":"active"},"drivews":{"pcsRequired":true,"url":"https://p10-drivews.icloud.com:443","status":"active"},"uploadimagews":{"url":"https://p10-uploadimagews.icloud.com:443","status":"active"},"schoolwork":{},"cksharews":{"url":"https://p10-ckshare.icloud.com:443","status":"active"},"findme":{"url":"https://p10-fmipweb.icloud.com:443","status":"active"},"ckdeviceservice":{"url":"https://p10-ckdevice.icloud.com:443"},"iworkthumbnailws":{"url":"https://p10-iworkthumbnailws.icloud.com:443","status":"active"},"calendar":{"url":"https://p10-calendarws.icloud.com:443","status":"active"},"docws":{"pcsRequired":true,"url":"https://p10-docws.icloud.com:443","status":"active"},"settings":{"url":"https://p10-settingsws.icloud.com:443","status":"active"},"ubiquity":{"url":"https://p10-ubiquityws.icloud.com:443","status":"active"},"streams":{"url":"https://p10-streams.icloud.com:443","status":"active"},"keyvalue":{"url":"https://p10-keyvalueservice.icloud.com:443","status":"active"},"archivews":{"url":"https://p10-archivews.icloud.com:443","status":"active"},"push":{"url":"https://p10-pushws.icloud.com:443","status":"active"},"iwmb":{"url":"https://p10-iwmb.icloud.com:443","status":"active"},"iworkexportws":{"url":"https://p10-iworkexportws.icloud.com:443","status":"active"},"geows":{"url":"https://p10-geows.icloud.com:443","status":"active"},"account":{"iCloudEnv":{"shortId":"p","vipSuffix":"p"},"url":"https://p10-setup.icloud.com:443","status":"active"},"fmf":{"url":"https://p10-fmfweb.icloud.com:443","status":"active"},"contacts":{"url":"https://p10-contactsws.icloud.com:443","status":"active"}},"pcsEnabled":true,"configBag":{"urls":{"accountCreateUI":"https://appleid.apple.com/widget/account/?widgetKey=d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d#!create","accountLoginUI":"https://idmsa.apple.com/appleauth/auth/signin?widgetKey=83545bf919730e51dbfba24e7e8a78d2","accountLogin":"https://setup.icloud.com/setup/ws/1/accountLogin","accountRepairUI":"https://appleid.apple.com/widget/account/?widgetKey=d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d#!repair","downloadICloudTerms":"https://setup.icloud.com/setup/ws/1/downloadLiteTerms","repairDone":"https://setup.icloud.com/setup/ws/1/repairDone","vettingUrlForEmail":"https://id.apple.com/IDMSEmailVetting/vetShareEmail","accountCreate":"https://setup.icloud.com/setup/ws/1/createLiteAccount","getICloudTerms":"https://setup.icloud.com/setup/ws/1/getTerms","vettingUrlForPhone":"https://id.apple.com/IDMSEmailVetting/vetSharePhone"},"accountCreateEnabled":"true"},"hsaTrustedBrowser":false,"appsOrder":["mail","contacts","calendar","photos","iclouddrive","notes2","reminders","pages","numbers","keynote","newspublisher","fmf","find","settings"],"version":2,"isExtendedLogin":false,"pcsServiceIdentitiesIncluded":false,"hsaChallengeRequired":true,"requestInfo":{"country":"TH","timeZone":"GMT+7"},"pcsDeleted":false,"iCloudInfo":{"SafariBookmarksHasMigratedToCloudKit":false},"apps":{"calendar":{},"reminders":{},"keynote":{"isQualifiedForBeta":true},"settings":{"canLaunchWithOneFactor":true},"mail":{},"numbers":{"isQualifiedForBeta":true},"photos":{},"pages":{"isQualifiedForBeta":true},"find":{"canLaunchWithOneFactor":true},"notes2":{},"iclouddrive":{},"newspublisher":{"isHidden":true},"fmf":{},"contacts":{}}}'} 18 | headers: 19 | Access-Control-Allow-Credentials: ['true'] 20 | Access-Control-Allow-Origin: ['https://www.icloud.com'] 21 | Apple-Originating-System: [UnknownOriginatingSystem] 22 | Cache-Control: ['no-cache, no-store, private'] 23 | Connection: [keep-alive] 24 | Content-Type: [application/json; charset=UTF-8] 25 | Date: ['Tue, 31 Jul 2018 09:54:33 GMT'] 26 | Server: [AppleHttpServer/2f080fc0] 27 | Strict-Transport-Security: [max-age=31536000; includeSubDomains] 28 | X-Apple-Jingle-Correlation-Key: [VNV7VU7BCBAU7G7ACJPP6JHPVY] 29 | X-Apple-Request-UUID: [ab6bfad3-e110-414f-9be0-125eff24efae] 30 | X-Responding-Instance: ['setupservice:11700403:st13p17ic-setupsvc004:8003:1813B80:3b85e7d76'] 31 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 32 | apple-seq: ['0'] 33 | apple-tk: ['false'] 34 | content-length: ['4872'] 35 | via: ['icloudedge:si03p01ic-ztde010418:7401:18RC341:Singapore'] 36 | status: {code: 200, message: OK} 37 | - request: 38 | body: null 39 | headers: 40 | Accept: ['*/*'] 41 | Accept-Encoding: ['gzip, deflate'] 42 | Connection: [keep-alive] 43 | Origin: ['https://www.icloud.com'] 44 | Referer: ['https://www.icloud.com/'] 45 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 46 | method: GET 47 | uri: https://setup.icloud.com/setup/ws/1/listDevices?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&ckjsBuildVersion=17DProjectDev77&ckjsVersion=2.0.5&clientId=DE309E26-942E-11E8-92F5-14109FE0B321&dsid=185776146 48 | response: 49 | body: {string: '{"devices":[{"deviceType":"SMS","areaCode":"","phoneNumber":"*******03","deviceId":"1"}]}'} 50 | headers: 51 | Access-Control-Allow-Credentials: ['true'] 52 | Access-Control-Allow-Origin: ['https://www.icloud.com'] 53 | Apple-Originating-System: [UnknownOriginatingSystem] 54 | Cache-Control: ['no-cache, no-store, private'] 55 | Connection: [keep-alive] 56 | Content-Type: [application/json; charset=UTF-8] 57 | Date: ['Tue, 31 Jul 2018 09:54:34 GMT'] 58 | Server: [AppleHttpServer/2f080fc0] 59 | Strict-Transport-Security: [max-age=31536000; includeSubDomains] 60 | X-Apple-Jingle-Correlation-Key: [HZN37F7FIVEQLMEB62ZL7MRV4Y] 61 | X-Apple-Request-UUID: [3e5bbf97-e545-4905-b081-f6b2bfb235e6] 62 | X-Responding-Instance: ['setupservice:21800401:nk11p18ic-setupsvc004:8001:1813B80:3b85e7d76'] 63 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 64 | apple-seq: ['0'] 65 | apple-tk: ['false'] 66 | content-length: ['89'] 67 | via: ['icloudedge:si03p01ic-ztde010418:7401:18RC341:Singapore'] 68 | status: {code: 200, message: OK} 69 | - request: 70 | body: '{"deviceType": "SMS", "areaCode": "", "phoneNumber": "*******03", "deviceId": 71 | "1"}' 72 | headers: 73 | Accept: ['*/*'] 74 | Accept-Encoding: ['gzip, deflate'] 75 | Connection: [keep-alive] 76 | Content-Length: ['82'] 77 | Origin: ['https://www.icloud.com'] 78 | Referer: ['https://www.icloud.com/'] 79 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 80 | method: POST 81 | uri: https://setup.icloud.com/setup/ws/1/sendVerificationCode?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&ckjsBuildVersion=17DProjectDev77&ckjsVersion=2.0.5&clientId=DE309E26-942E-11E8-92F5-14109FE0B321&dsid=185776146 82 | response: 83 | body: {string: '{"success":true}'} 84 | headers: 85 | Access-Control-Allow-Credentials: ['true'] 86 | Access-Control-Allow-Origin: ['https://www.icloud.com'] 87 | Apple-Originating-System: [UnknownOriginatingSystem] 88 | Cache-Control: ['no-cache, no-store, private'] 89 | Connection: [keep-alive] 90 | Content-Type: [application/json; charset=UTF-8] 91 | Date: ['Tue, 31 Jul 2018 09:54:35 GMT'] 92 | Server: [AppleHttpServer/2f080fc0] 93 | Strict-Transport-Security: [max-age=31536000; includeSubDomains] 94 | X-Apple-Jingle-Correlation-Key: [CWUBPOMXINCETCSF3QVGN44BR4] 95 | X-Apple-Request-UUID: [15a817b9-9743-4449-8a45-dc2a66f3818f] 96 | X-Responding-Instance: ['setupservice:34400103:mr26p44ic-tydg07122301:8003:1813B80:3b85e7d76'] 97 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 98 | apple-seq: ['0'] 99 | apple-tk: ['false'] 100 | content-length: ['16'] 101 | via: ['icloudedge:si03p01ic-ztde010418:7401:18RC341:Singapore'] 102 | status: {code: 200, message: OK} 103 | - request: 104 | body: '{"verificationCode": "901431", "trustBrowser": true}' 105 | headers: 106 | Accept: ['*/*'] 107 | Accept-Encoding: ['gzip, deflate'] 108 | Connection: [keep-alive] 109 | Content-Length: ['52'] 110 | Origin: ['https://www.icloud.com'] 111 | Referer: ['https://www.icloud.com/'] 112 | User-Agent: [Opera/9.52 (X11; Linux i686; U; en)] 113 | method: POST 114 | uri: https://setup.icloud.com/setup/ws/1/validateVerificationCode?clientBuildNumber=17DHotfix5&clientMasteringNumber=17DHotfix5&ckjsBuildVersion=17DProjectDev77&ckjsVersion=2.0.5&clientId=DE309E26-942E-11E8-92F5-14109FE0B321&dsid=185776146 115 | response: 116 | body: {string: '{"success":false,"errorTitle":"Invalid Code","errorMessage":"The 117 | code you entered is not valid.","errorCode":-21669}'} 118 | headers: 119 | Access-Control-Allow-Credentials: ['true'] 120 | Access-Control-Allow-Origin: ['https://www.icloud.com'] 121 | Apple-Originating-System: [UnknownOriginatingSystem] 122 | Cache-Control: ['no-cache, no-store, private'] 123 | Connection: [keep-alive] 124 | Content-Type: [application/json; charset=UTF-8] 125 | Date: ['Tue, 31 Jul 2018 09:57:30 GMT'] 126 | Server: [AppleHttpServer/2f080fc0] 127 | Strict-Transport-Security: [max-age=31536000; includeSubDomains] 128 | X-Apple-Jingle-Correlation-Key: [ZXDIZFYF7VC2VFMSYPEOKPT6CM] 129 | X-Apple-Request-UUID: [cdc68c97-05fd-45aa-9592-c3c8e53e7e13] 130 | X-Responding-Instance: ['setupservice:34600403:mr25p46ic-tydg01141201:8003:1813B80:3b85e7d76'] 131 | access-control-expose-headers: [X-Apple-Request-UUID, Via] 132 | apple-seq: ['0'] 133 | apple-tk: ['false'] 134 | content-length: ['116'] 135 | via: ['icloudedge:si03p01ic-ztde010418:7401:18RC341:Singapore'] 136 | status: {code: 200, message: OK} 137 | version: 1 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ndbroadbent/icloud_photos_downloader.svg?branch=master)](https://travis-ci.org/ndbroadbent/icloud_photos_downloader) 2 | [![Coverage Status](https://coveralls.io/repos/github/ndbroadbent/icloud_photos_downloader/badge.svg?branch=master)](https://coveralls.io/github/ndbroadbent/icloud_photos_downloader?branch=master) 3 | [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) 4 | 5 | # iCloud Photos Downloader 6 | 7 | - A command-line tool to download all your iCloud photos. 8 | - Works on Linux, Windows, and MacOS. 9 | - Run as a [scheduled cron task](#cron-task) to keep a local backup of your photos and videos. 10 | 11 | ## Install 12 | 13 | `icloudpd` is a Python package that can be installed using `pip`: 14 | 15 | ``` 16 | pip install icloudpd 17 | ``` 18 | 19 | > If you need to install Python, see the [Requirements](#requirements) section for instructions. 20 | 21 | ## Usage 22 | 23 | $ icloudpd 24 | --username 25 | [--password ] 26 | [--cookie-directory ] 27 | [--size (original|medium|thumb)] 28 | [--live-photo-size (original|medium|thumb)] 29 | [--recent ] 30 | [--until-found ] 31 | [--skip-videos] 32 | [--skip-live-photos] 33 | [--force-size] 34 | [--auto-delete] 35 | [--only-print-filenames] 36 | [--folder-structure ({:%Y/%m/%d})] 37 | [--set-exif-datetime] 38 | [--smtp-username ] 39 | [--smtp-password ] 40 | [--smtp-host ] 41 | [--smtp-port ] 42 | [--smtp-no-tls] 43 | [--notification-email ] 44 | [--notification-script PATH] 45 | [--log-level (debug|info|error)] 46 | [--no-progress-bar] 47 | 48 | Options: 49 | --username Your iCloud username or email address 50 | --password Your iCloud password (default: use PyiCloud 51 | keyring or prompt for password) 52 | --cookie-directory 53 | Directory to store cookies for 54 | authentication (default: ~/.pyicloud) 55 | --size [original|medium|thumb] Image size to download (default: original) 56 | --live-photo-size [original|medium|thumb] 57 | Live Photo video size to download (default: 58 | original) 59 | --recent INTEGER RANGE Number of recent photos to download 60 | (default: download all photos) 61 | --until-found INTEGER RANGE Download most recently added photos until we 62 | find x number of previously downloaded 63 | consecutive photos (default: download all 64 | photos) 65 | --skip-videos Don't download any videos (default: Download 66 | both photos and videos) 67 | --skip-live-photos Don't download any live photos (default: 68 | Download live photos) 69 | --force-size Only download the requested size (default: 70 | download original if size is not available) 71 | --auto-delete Scans the "Recently Deleted" folder and 72 | deletes any files found in there. (If you 73 | restore the photo in iCloud, it will be 74 | downloaded again.) 75 | --only-print-filenames Only prints the filenames of all files that 76 | will be downloaded. (Does not download any 77 | files.) 78 | --folder-structure 79 | Folder structure (default: {:%Y/%m/%d}) 80 | --set-exif-datetime Write the DateTimeOriginal exif tag from 81 | file creation date, if it doesn't exist. 82 | --smtp-username 83 | Your SMTP username, for sending email 84 | notifications when two-step authentication 85 | expires. 86 | --smtp-password 87 | Your SMTP password, for sending email 88 | notifications when two-step authentication 89 | expires. 90 | --smtp-host Your SMTP server host. Defaults to: 91 | smtp.gmail.com 92 | --smtp-port Your SMTP server port. Default: 587 (Gmail) 93 | --smtp-no-tls Pass this flag to disable TLS for SMTP (TLS 94 | is required for Gmail) 95 | --notification-email 96 | Email address where you would like to 97 | receive email notifications. Default: SMTP 98 | username 99 | --notification-script PATH Runs an external script when two factor 100 | authentication expires. (path required: 101 | /path/to/my/script.sh) 102 | --log-level [debug|info|error] Log level (default: debug) 103 | --no-progress-bar Disables the one-line progress bar and 104 | prints log messages on separate lines 105 | (Progress bar is disabled by default if 106 | there is no tty attached) 107 | --version Show the version and exit. 108 | -h, --help Show this message and exit. 109 | 110 | Example: 111 | 112 | $ icloudpd ./Photos \ 113 | --username testuser@example.com \ 114 | --password pass1234 \ 115 | --recent 500 \ 116 | --auto-delete 117 | 118 | ## Requirements 119 | 120 | - Python 2.7 or Python 3.4+ 121 | - _Python 2.6 is not supported._ 122 | - pip 123 | 124 | ### Install Python & pip 125 | 126 | #### Windows 127 | 128 | - [Download Python 3.7.0](https://www.python.org/ftp/python/3.7.0/python-3.7.0.exe) 129 | 130 | #### Mac 131 | 132 | - Install [Homebrew](https://brew.sh/) (if not already installed): 133 | 134 | ``` 135 | which brew > /dev/null 2>&1 || /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 136 | ``` 137 | 138 | - Install Python (includes `pip`): 139 | 140 | ``` 141 | brew install python 142 | ``` 143 | 144 | > Alternatively, you can [download the Python 3.7.0 installer for Mac](https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg). 145 | 146 | #### Linux (Ubuntu) 147 | 148 | ``` 149 | sudo apt-get update 150 | sudo apt-get install -y python 151 | ``` 152 | 153 | ## Authentication 154 | 155 | If your Apple account has two-factor authentication enabled, 156 | you will be prompted for a code when you run the script. 157 | 158 | Two-factor authentication will expire after an interval set by Apple, 159 | at which point you will have to re-authenticate. This interval is currently two months. 160 | 161 | Authentication cookies will be stored in a temp directory (`/tmp/pyicloud` on Linux, or `/var/tmp/...` on MacOS.) This directory can be configured with the `--cookie-directory` option. 162 | 163 | You can receive an email notification when two-factor authentication expires by passing the 164 | `--smtp-username` and `--smtp-password` options. Emails will be sent to `--smtp-username` by default, 165 | or you can send to a different email address with `--notification-email`. 166 | 167 | If you want to send notification emails using your Gmail account, and you have enabled two-factor authentication, you will need to generate an App Password at https://myaccount.google.com/apppasswords 168 | 169 | ### System Keyring 170 | 171 | You can store your password in the system keyring using the `icloud` command-line tool 172 | (installed with the `pyicloud` dependency): 173 | 174 | $ icloud --username jappleseed@apple.com 175 | ICloud Password for jappleseed@apple.com: 176 | Save password in keyring? (y/N) 177 | 178 | If you have stored a password in the keyring, you will not be required to provide a password 179 | when running the script. 180 | 181 | If you would like to delete a password stored in your system keyring, 182 | you can clear a stored password using the `--delete-from-keyring` command-line option: 183 | 184 | $ icloud --username jappleseed@apple.com --delete-from-keyring 185 | 186 | ## Error on first run 187 | 188 | When you run the script for the first time, you might see an error message like this: 189 | 190 | ``` 191 | Bad Request (400) 192 | ``` 193 | 194 | This error often happens because your account hasn't used the iCloud API before, so Apple's servers need to prepare some information about your photos. This process can take around 5-10 minutes, so please wait a few minutes and try again. 195 | 196 | If you are still seeing this message after 30 minutes, then please [open an issue on GitHub](https://github.com/ndbroadbent/icloud_photos_downloader/issues/new) and post the script output. 197 | 198 | ## Cron Task 199 | 200 | Follow these instructions to run `icloudpd` as a scheduled cron task. 201 | 202 | ``` 203 | # Clone the git repo somewhere 204 | git clone https://github.com/ndbroadbent/icloud_photos_downloader.git 205 | cd icloud_photos_downloader 206 | 207 | # Copy the example cron script 208 | cp cron_script.sh.example cron_script.sh 209 | ``` 210 | 211 | - Update `cron_script.sh` with your username, password, and other options 212 | 213 | - Edit your "crontab" with `crontab -e`, then add the following line: 214 | 215 | ``` 216 | 0 */6 * * * /path/to/icloud_photos_downloader/cron_script.sh 217 | ``` 218 | 219 | Now the script will run every 6 hours to download any new photos and videos. 220 | 221 | > If you provide SMTP credentials, the script will send an email notification 222 | > whenever two-step authentication expires. 223 | 224 | ## Docker 225 | 226 | This script is available in a Docker image: `docker pull ndbroadbent/icloudpd` 227 | 228 | Usage: 229 | 230 | ```bash 231 | # Downloads all photos to ./Photos 232 | 233 | $ docker pull ndbroadbent/icloudpd 234 | $ docker run -it --rm --name icloud -v $(pwd)/Photos:/data ndbroadbent/icloudpd:latest \ 235 | icloudpd /data \ 236 | --username testuser@example.com \ 237 | --password pass1234 \ 238 | --size original \ 239 | --recent 500 \ 240 | --auto-delete 241 | ``` 242 | 243 | ## Contributing 244 | 245 | Install dependencies: 246 | 247 | ``` 248 | sudo pip install -r requirements.txt 249 | sudo pip install -r requirements-test.txt 250 | ``` 251 | 252 | Run tests: 253 | 254 | ``` 255 | pytest 256 | ``` 257 | 258 | Before submitting a pull request, please check the following: 259 | 260 | - All tests pass on Python 2.7 and 3.6 261 | - Run `./scripts/test` 262 | - 100% test coverage 263 | - After running `./scripts/test`, you will see the test coverage results in the output 264 | - You can also open the HTML report at: `./htmlcov/index.html` 265 | - Code is formatted with [autopep8](https://github.com/hhatto/autopep8) 266 | - Run `./scripts/format` 267 | - No [pylint](https://www.pylint.org/) errors 268 | - Run `./scripts/lint` (or `pylint icloudpd`) 269 | - If you've added or changed any command-line options, 270 | please update the [Usage](#usage) section in the README. 271 | 272 | If you need to make any changes to the `pyicloud` library, 273 | `icloudpd` uses a fork of this library that has been renamed to `pyicloud-ipd`. 274 | Please clone my [pyicloud fork](https://github.com/ndbroadbent/pyicloud) 275 | and check out the [pyicloud-ipd](https://github.com/ndbroadbent/pyicloud/tree/pyicloud-ipd) 276 | branch. PRs should be based on the `pyicloud-ipd` branch and submitted to 277 | [ndbroadbent/pyicloud](https://github.com/ndbroadbent/pyicloud). 278 | 279 | ### Building the Docker image: 280 | 281 | ``` 282 | $ git clone https://github.com/ndbroadbent/icloud_photos_downloader.git 283 | $ cd icloud_photos_downloader/docker 284 | $ docker build -t ndbroadbent/icloudpd . 285 | ``` 286 | 287 | ## Support Development 288 | 289 | Buy Me a Coffee 290 | 291 | Buy Me a Coffee 292 | 293 | - _Bitcoin (BTC)_: 15eSc6JiwBtxte9zak44ZFhw9bkKWaMGAe 294 | - _Ethereum (ETH)_: 0x080300A117758EC601b7b6c501afE3571E5cA1c3 295 | -------------------------------------------------------------------------------- /icloudpd/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Main script that uses Click to parse command-line arguments""" 3 | from __future__ import print_function 4 | import os 5 | import sys 6 | import time 7 | import datetime 8 | import logging 9 | import itertools 10 | import subprocess 11 | import json 12 | import click 13 | 14 | from tqdm import tqdm 15 | from tzlocal import get_localzone 16 | 17 | from icloudpd.logger import setup_logger 18 | from icloudpd.authentication import authenticate, TwoStepAuthRequiredError 19 | from icloudpd import download 20 | from icloudpd.email_notifications import send_2sa_notification 21 | from icloudpd.string_helpers import truncate_middle 22 | from icloudpd.autodelete import autodelete_photos 23 | from icloudpd.paths import local_download_path 24 | from icloudpd import exif_datetime 25 | # Must import the constants object so that we can mock values in tests. 26 | from icloudpd import constants 27 | 28 | CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) 29 | 30 | 31 | @click.command(context_settings=CONTEXT_SETTINGS, options_metavar="") 32 | @click.argument( 33 | "directory", 34 | type=click.Path( 35 | exists=True), 36 | metavar="") 37 | @click.option( 38 | "--username", 39 | help="Your iCloud username or email address", 40 | metavar="", 41 | prompt="iCloud username/email", 42 | ) 43 | @click.option( 44 | "--password", 45 | help="Your iCloud password " 46 | "(default: use PyiCloud keyring or prompt for password)", 47 | metavar="", 48 | ) 49 | @click.option( 50 | "--cookie-directory", 51 | help="Directory to store cookies for authentication " 52 | "(default: ~/.pyicloud)", 53 | metavar="", 54 | default="~/.pyicloud", 55 | ) 56 | @click.option( 57 | "--size", 58 | help="Image size to download (default: original)", 59 | type=click.Choice(["original", "medium", "thumb"]), 60 | default="original", 61 | ) 62 | @click.option( 63 | "--live-photo-size", 64 | help="Live Photo video size to download (default: original)", 65 | type=click.Choice(["original", "medium", "thumb"]), 66 | default="original", 67 | ) 68 | @click.option( 69 | "--recent", 70 | help="Number of recent photos to download (default: download all photos)", 71 | type=click.IntRange(0), 72 | ) 73 | @click.option( 74 | "--until-found", 75 | help="Download most recently added photos until we find x number of " 76 | "previously downloaded consecutive photos (default: download all photos)", 77 | type=click.IntRange(0), 78 | ) 79 | @click.option( 80 | "--skip-videos", 81 | help="Don't download any videos (default: Download all photos and videos)", 82 | is_flag=True, 83 | ) 84 | @click.option( 85 | "--skip-live-photos", 86 | help="Don't download any live photos (default: Download live photos)", 87 | is_flag=True, 88 | ) 89 | @click.option( 90 | "--force-size", 91 | help="Only download the requested size " 92 | + "(default: download original if size is not available)", 93 | is_flag=True, 94 | ) 95 | @click.option( 96 | "--auto-delete", 97 | help='Scans the "Recently Deleted" folder and deletes any files found in there. ' 98 | + "(If you restore the photo in iCloud, it will be downloaded again.)", 99 | is_flag=True, 100 | ) 101 | @click.option( 102 | "--only-print-filenames", 103 | help="Only prints the filenames of all files that will be downloaded " 104 | "(not including files that are already downloaded.)" 105 | + "(Does not download or delete any files.)", 106 | is_flag=True, 107 | ) 108 | @click.option( 109 | "--folder-structure", 110 | help="Folder structure (default: {:%Y/%m/%d})", 111 | metavar="", 112 | default="{:%Y/%m/%d}", 113 | ) 114 | @click.option( 115 | "--set-exif-datetime", 116 | help="Write the DateTimeOriginal exif tag from file creation date, if it doesn't exist.", 117 | is_flag=True, 118 | ) 119 | @click.option( 120 | "--smtp-username", 121 | help="Your SMTP username, for sending email notifications when " 122 | "two-step authentication expires.", 123 | metavar="", 124 | ) 125 | @click.option( 126 | "--smtp-password", 127 | help="Your SMTP password, for sending email notifications when " 128 | "two-step authentication expires.", 129 | metavar="", 130 | ) 131 | @click.option( 132 | "--smtp-host", 133 | help="Your SMTP server host. Defaults to: smtp.gmail.com", 134 | metavar="", 135 | default="smtp.gmail.com", 136 | ) 137 | @click.option( 138 | "--smtp-port", 139 | help="Your SMTP server port. Default: 587 (Gmail)", 140 | metavar="", 141 | type=click.IntRange(0), 142 | default=587, 143 | ) 144 | @click.option( 145 | "--smtp-no-tls", 146 | help="Pass this flag to disable TLS for SMTP (TLS is required for Gmail)", 147 | metavar="", 148 | is_flag=True, 149 | ) 150 | @click.option( 151 | "--notification-email", 152 | help="Email address where you would like to receive email notifications. " 153 | "Default: SMTP username", 154 | metavar="", 155 | ) 156 | @click.option( 157 | "--notification-script", 158 | type=click.Path(), 159 | help="Runs an external script when two factor authentication expires. " 160 | "(path required: /path/to/my/script.sh)", 161 | ) 162 | @click.option( 163 | "--log-level", 164 | help="Log level (default: debug)", 165 | type=click.Choice(["debug", "info", "error"]), 166 | default="debug", 167 | ) 168 | @click.option( 169 | "--no-progress-bar", 170 | help="Disables the one-line progress bar and prints log messages on separate lines " 171 | "(Progress bar is disabled by default if there is no tty attached)", 172 | is_flag=True, 173 | ) 174 | @click.version_option() 175 | # pylint: disable-msg=too-many-arguments,too-many-statements 176 | # pylint: disable-msg=too-many-branches,too-many-locals 177 | def main( 178 | directory, 179 | username, 180 | password, 181 | cookie_directory, 182 | size, 183 | live_photo_size, 184 | recent, 185 | until_found, 186 | skip_videos, 187 | skip_live_photos, 188 | force_size, 189 | auto_delete, 190 | only_print_filenames, 191 | folder_structure, 192 | set_exif_datetime, 193 | smtp_username, 194 | smtp_password, 195 | smtp_host, 196 | smtp_port, 197 | smtp_no_tls, 198 | notification_email, 199 | log_level, 200 | no_progress_bar, 201 | notification_script, 202 | ): 203 | """Download all iCloud photos to a local directory""" 204 | logger = setup_logger() 205 | if only_print_filenames: 206 | logger.disabled = True 207 | else: 208 | # Need to make sure disabled is reset to the correct value, 209 | # because the logger instance is shared between tests. 210 | logger.disabled = False 211 | if log_level == "debug": 212 | logger.setLevel(logging.DEBUG) 213 | elif log_level == "info": 214 | logger.setLevel(logging.INFO) 215 | elif log_level == "error": 216 | logger.setLevel(logging.ERROR) 217 | 218 | raise_error_on_2sa = ( 219 | smtp_username is not None 220 | or notification_email is not None 221 | or notification_script is not None 222 | ) 223 | try: 224 | icloud = authenticate( 225 | username, 226 | password, 227 | cookie_directory, 228 | raise_error_on_2sa, 229 | client_id=os.environ.get("CLIENT_ID"), 230 | ) 231 | except TwoStepAuthRequiredError: 232 | if notification_script is not None: 233 | subprocess.call([notification_script]) 234 | if smtp_username is not None or notification_email is not None: 235 | send_2sa_notification( 236 | smtp_username, 237 | smtp_password, 238 | smtp_host, 239 | smtp_port, 240 | smtp_no_tls, 241 | notification_email, 242 | ) 243 | exit(1) 244 | 245 | # For Python 2.7 246 | if hasattr(directory, "decode"): 247 | directory = directory.decode("utf-8") # pragma: no cover 248 | directory = os.path.normpath(directory) 249 | 250 | logger.debug( 251 | "Looking up all photos%s...", 252 | "" if skip_videos else " and videos") 253 | photos = icloud.photos.all 254 | 255 | def photos_exception_handler(ex, retries): 256 | """Handles session errors in the PhotoAlbum photos iterator""" 257 | if "Invalid global session" in str(ex): 258 | if retries > constants.MAX_RETRIES: 259 | logger.tqdm_write( 260 | "iCloud re-authentication failed! Please try again later." 261 | ) 262 | raise ex 263 | logger.tqdm_write( 264 | "Session error, re-authenticating...", 265 | logging.ERROR) 266 | if retries > 1: 267 | # If the first reauthentication attempt failed, 268 | # start waiting a few seconds before retrying in case 269 | # there are some issues with the Apple servers 270 | time.sleep(constants.WAIT_SECONDS) 271 | icloud.authenticate() 272 | 273 | photos.exception_handler = photos_exception_handler 274 | 275 | photos_count = len(photos) 276 | 277 | # Optional: Only download the x most recent photos. 278 | if recent is not None: 279 | photos_count = recent 280 | photos = itertools.islice(photos, recent) 281 | 282 | tqdm_kwargs = {"total": photos_count} 283 | 284 | if until_found is not None: 285 | del tqdm_kwargs["total"] 286 | photos_count = "???" 287 | # ensure photos iterator doesn't have a known length 288 | photos = (p for p in photos) 289 | 290 | plural_suffix = "" if photos_count == 1 else "s" 291 | video_suffix = "" 292 | photos_count_str = "the first" if photos_count == 1 else photos_count 293 | if not skip_videos: 294 | video_suffix = " or video" if photos_count == 1 else " and videos" 295 | logger.info( 296 | "Downloading %s %s photo%s%s to %s/ ...", 297 | photos_count_str, 298 | size, 299 | plural_suffix, 300 | video_suffix, 301 | directory, 302 | ) 303 | 304 | consecutive_files_found = 0 305 | 306 | # Use only ASCII characters in progress bar 307 | tqdm_kwargs["ascii"] = True 308 | 309 | # Skip the one-line progress bar if we're only printing the filenames, 310 | # or if the progress bar is explicity disabled, 311 | # or if this is not a terminal (e.g. cron or piping output to file) 312 | if not os.environ.get("FORCE_TQDM") and ( 313 | only_print_filenames or no_progress_bar or not sys.stdout.isatty() 314 | ): 315 | photos_enumerator = photos 316 | logger.set_tqdm(None) 317 | else: 318 | photos_enumerator = tqdm(photos, **tqdm_kwargs) 319 | logger.set_tqdm(photos_enumerator) 320 | 321 | # pylint: disable-msg=too-many-nested-blocks 322 | for photo in photos_enumerator: 323 | for _ in range(constants.MAX_RETRIES): 324 | if skip_videos and photo.item_type != "image": 325 | logger.set_tqdm_description( 326 | "Skipping %s, only downloading photos." % photo.filename 327 | ) 328 | break 329 | if photo.item_type != "image" and photo.item_type != "movie": 330 | logger.set_tqdm_description( 331 | "Skipping %s, only downloading photos and videos. " 332 | "(Item type was: %s)" % (photo.filename, photo.item_type) 333 | ) 334 | break 335 | try: 336 | created_date = photo.created.astimezone(get_localzone()) 337 | except (ValueError, OSError): 338 | logger.set_tqdm_description( 339 | "Could not convert photo created date to local timezone (%s)" % 340 | photo.created, logging.ERROR) 341 | created_date = photo.created 342 | 343 | try: 344 | date_path = folder_structure.format(created_date) 345 | except ValueError: # pragma: no cover 346 | # This error only seems to happen in Python 2 347 | logger.set_tqdm_description( 348 | "Photo created date was not valid (%s)" % 349 | photo.created, logging.ERROR) 350 | # e.g. ValueError: year=5 is before 1900 351 | # (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122) 352 | # Just use the Unix epoch 353 | created_date = datetime.datetime.fromtimestamp(0) 354 | date_path = folder_structure.format(created_date) 355 | 356 | download_dir = os.path.join(directory, date_path) 357 | 358 | if not os.path.exists(download_dir): 359 | os.makedirs(download_dir) 360 | 361 | download_size = size 362 | 363 | try: 364 | versions = photo.versions 365 | except KeyError as ex: 366 | print( 367 | "KeyError: %s attribute was not found in the photo fields!" % 368 | ex) 369 | with open('icloudpd-photo-error.json', 'w') as outfile: 370 | # pylint: disable=protected-access 371 | json.dump({ 372 | "master_record": photo._master_record, 373 | "asset_record": photo._asset_record 374 | }, outfile) 375 | # pylint: enable=protected-access 376 | print("icloudpd has saved the photo record to: " 377 | "./icloudpd-photo-error.json") 378 | print("Please create a Gist with the contents of this file: " 379 | "https://gist.github.com") 380 | print( 381 | "Then create an issue on GitHub: " 382 | "https://github.com/ndbroadbent/icloud_photos_downloader/issues") 383 | print( 384 | "Include a link to the Gist in your issue, so that we can " 385 | "see what went wrong.\n") 386 | break 387 | 388 | if size not in versions and size != "original": 389 | if force_size: 390 | filename = photo.filename.encode( 391 | "utf-8").decode("ascii", "ignore") 392 | logger.set_tqdm_description( 393 | "%s size does not exist for %s. Skipping..." % 394 | (size, filename), logging.ERROR, ) 395 | break 396 | download_size = "original" 397 | 398 | download_path = local_download_path( 399 | photo, download_size, download_dir) 400 | 401 | file_exists = os.path.isfile(download_path) 402 | if not file_exists and download_size == "original": 403 | # Deprecation - We used to download files like IMG_1234-original.jpg, 404 | # so we need to check for these. 405 | # Now we match the behavior of iCloud for Windows: IMG_1234.jpg 406 | original_download_path = ("-%s." % size).join( 407 | download_path.rsplit(".", 1) 408 | ) 409 | file_exists = os.path.isfile(original_download_path) 410 | 411 | if file_exists: 412 | if until_found is not None: 413 | consecutive_files_found += 1 414 | logger.set_tqdm_description( 415 | "%s already exists." % truncate_middle(download_path, 96) 416 | ) 417 | else: 418 | if until_found is not None: 419 | consecutive_files_found = 0 420 | 421 | if only_print_filenames: 422 | print(download_path) 423 | else: 424 | truncated_path = truncate_middle(download_path, 96) 425 | logger.set_tqdm_description( 426 | "Downloading %s" % 427 | truncated_path) 428 | 429 | download_result = download.download_media( 430 | icloud, photo, download_path, download_size 431 | ) 432 | 433 | if download_result and set_exif_datetime: 434 | if photo.filename.lower().endswith((".jpg", ".jpeg")): 435 | if not exif_datetime.get_photo_exif(download_path): 436 | # %Y:%m:%d looks wrong but it's the correct format 437 | date_str = created_date.strftime( 438 | "%Y:%m:%d %H:%M:%S") 439 | logger.debug( 440 | "Setting EXIF timestamp for %s: %s", 441 | download_path, 442 | date_str, 443 | ) 444 | exif_datetime.set_photo_exif( 445 | download_path, 446 | created_date.strftime("%Y:%m:%d %H:%M:%S"), 447 | ) 448 | else: 449 | timestamp = time.mktime(created_date.timetuple()) 450 | os.utime(download_path, (timestamp, timestamp)) 451 | 452 | # Also download the live photo if present 453 | if not skip_live_photos: 454 | lp_size = live_photo_size + "Video" 455 | if lp_size in photo.versions: 456 | version = photo.versions[lp_size] 457 | filename = version["filename"] 458 | if live_photo_size != "original": 459 | # Add size to filename if not original 460 | filename = filename.replace( 461 | ".MOV", "-%s.MOV" % 462 | live_photo_size) 463 | lp_download_path = os.path.join(download_dir, filename) 464 | 465 | if only_print_filenames: 466 | print(lp_download_path) 467 | else: 468 | if os.path.isfile(lp_download_path): 469 | logger.set_tqdm_description( 470 | "%s already exists." 471 | % truncate_middle(lp_download_path, 96) 472 | ) 473 | break 474 | 475 | truncated_path = truncate_middle(lp_download_path, 96) 476 | logger.set_tqdm_description( 477 | "Downloading %s" % truncated_path) 478 | download.download_media( 479 | icloud, photo, lp_download_path, lp_size 480 | ) 481 | 482 | break 483 | 484 | if until_found is not None and consecutive_files_found >= until_found: 485 | logger.tqdm_write( 486 | "Found %d consecutive previously downloaded photos. Exiting" 487 | % until_found 488 | ) 489 | if hasattr(photos_enumerator, "close"): 490 | photos_enumerator.close() 491 | break 492 | 493 | if only_print_filenames: 494 | exit(0) 495 | 496 | logger.info("All photos have been downloaded!") 497 | 498 | if auto_delete: 499 | autodelete_photos(icloud, folder_structure, directory) 500 | -------------------------------------------------------------------------------- /tests/test_download_photos.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from vcr import VCR 3 | import os 4 | import sys 5 | import shutil 6 | import logging 7 | import click 8 | import pytest 9 | import mock 10 | import tzlocal 11 | import datetime 12 | from mock import call, ANY 13 | from click.testing import CliRunner 14 | import piexif 15 | from piexif._exceptions import InvalidImageDataError 16 | from pyicloud_ipd.services.photos import PhotoAsset, PhotoAlbum 17 | from pyicloud_ipd.base import PyiCloudService 18 | from pyicloud_ipd.exceptions import PyiCloudAPIResponseError 19 | from requests.exceptions import ConnectionError 20 | from icloudpd.base import main 21 | import icloudpd.constants 22 | from tests.helpers.print_result_exception import print_result_exception 23 | 24 | vcr = VCR(decode_compressed_response=True) 25 | 26 | class DownloadPhotoTestCase(TestCase): 27 | @pytest.fixture(autouse=True) 28 | def inject_fixtures(self, caplog): 29 | self._caplog = caplog 30 | 31 | def test_download_and_skip_existing_photos(self): 32 | if os.path.exists("tests/fixtures/Photos"): 33 | shutil.rmtree("tests/fixtures/Photos") 34 | os.makedirs("tests/fixtures/Photos") 35 | 36 | os.makedirs("tests/fixtures/Photos/2018/07/30/") 37 | open("tests/fixtures/Photos/2018/07/30/IMG_7408.JPG", "a").close() 38 | open("tests/fixtures/Photos/2018/07/30/IMG_7407.JPG", "a").close() 39 | 40 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 41 | # Pass fixed client ID via environment variable 42 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 43 | runner = CliRunner() 44 | result = runner.invoke( 45 | main, 46 | [ 47 | "--username", 48 | "jdoe@gmail.com", 49 | "--password", 50 | "password1", 51 | "--recent", 52 | "5", 53 | "--skip-videos", 54 | "--skip-live-photos", 55 | "--set-exif-datetime", 56 | "--no-progress-bar", 57 | "tests/fixtures/Photos", 58 | ], 59 | ) 60 | print_result_exception(result) 61 | 62 | self.assertIn("DEBUG Looking up all photos...", self._caplog.text) 63 | self.assertIn( 64 | "INFO Downloading 5 original photos to tests/fixtures/Photos/ ...", 65 | self._caplog.text, 66 | ) 67 | self.assertIn( 68 | "INFO Downloading tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 69 | self._caplog.text, 70 | ) 71 | self.assertNotIn( 72 | "IMG_7409.MOV", 73 | self._caplog.text, 74 | ) 75 | self.assertIn( 76 | "INFO tests/fixtures/Photos/2018/07/30/IMG_7408.JPG already exists.", 77 | self._caplog.text, 78 | ) 79 | self.assertIn( 80 | "INFO tests/fixtures/Photos/2018/07/30/IMG_7407.JPG already exists.", 81 | self._caplog.text, 82 | ) 83 | self.assertIn( 84 | "INFO Skipping IMG_7405.MOV, only downloading photos.", 85 | self._caplog.text, 86 | ) 87 | self.assertIn( 88 | "INFO Skipping IMG_7404.MOV, only downloading photos.", 89 | self._caplog.text, 90 | ) 91 | self.assertIn( 92 | "INFO All photos have been downloaded!", self._caplog.text 93 | ) 94 | 95 | # Check that file was downloaded 96 | self.assertTrue( 97 | os.path.exists("tests/fixtures/Photos/2018/07/31/IMG_7409.JPG")) 98 | # Check that mtime was updated to the photo creation date 99 | photo_mtime = os.path.getmtime("tests/fixtures/Photos/2018/07/31/IMG_7409.JPG") 100 | photo_modified_time = datetime.datetime.utcfromtimestamp(photo_mtime) 101 | self.assertEquals( 102 | "2018-07-31 07:22:24", 103 | photo_modified_time.strftime('%Y-%m-%d %H:%M:%S')) 104 | 105 | assert result.exit_code == 0 106 | 107 | def test_download_photos_and_set_exif(self): 108 | if os.path.exists("tests/fixtures/Photos"): 109 | shutil.rmtree("tests/fixtures/Photos") 110 | os.makedirs("tests/fixtures/Photos") 111 | 112 | os.makedirs("tests/fixtures/Photos/2018/07/30/") 113 | open("tests/fixtures/Photos/2018/07/30/IMG_7408.JPG", "a").close() 114 | open("tests/fixtures/Photos/2018/07/30/IMG_7407.JPG", "a").close() 115 | 116 | # Download the first photo, but mock the video download 117 | orig_download = PhotoAsset.download 118 | 119 | def mocked_download(self, size): 120 | if not hasattr(PhotoAsset, "already_downloaded"): 121 | response = orig_download(self, size) 122 | setattr(PhotoAsset, "already_downloaded", True) 123 | return response 124 | return mock.MagicMock() 125 | 126 | with mock.patch.object(PhotoAsset, "download", new=mocked_download): 127 | with mock.patch( 128 | "icloudpd.exif_datetime.get_photo_exif" 129 | ) as get_exif_patched: 130 | get_exif_patched.return_value = False 131 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 132 | # Pass fixed client ID via environment variable 133 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 134 | runner = CliRunner() 135 | result = runner.invoke( 136 | main, 137 | [ 138 | "--username", 139 | "jdoe@gmail.com", 140 | "--password", 141 | "password1", 142 | "--recent", 143 | "4", 144 | "--set-exif-datetime", 145 | # '--skip-videos', 146 | # "--skip-live-photos", 147 | "--no-progress-bar", 148 | "tests/fixtures/Photos", 149 | ], 150 | ) 151 | print_result_exception(result) 152 | 153 | self.assertIn( 154 | "DEBUG Looking up all photos and videos...", 155 | self._caplog.text, 156 | ) 157 | self.assertIn( 158 | "INFO Downloading 4 original photos and videos to tests/fixtures/Photos/ ...", 159 | self._caplog.text, 160 | ) 161 | self.assertIn( 162 | "INFO Downloading tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 163 | self._caplog.text, 164 | ) 165 | # YYYY:MM:DD is the correct format. 166 | self.assertIn( 167 | "DEBUG Setting EXIF timestamp for tests/fixtures/Photos/2018/07/31/IMG_7409.JPG: 2018:07:31", 168 | self._caplog.text, 169 | ) 170 | self.assertIn( 171 | "INFO All photos have been downloaded!", self._caplog.text 172 | ) 173 | assert result.exit_code == 0 174 | 175 | def test_download_photos_and_exif_exceptions(self): 176 | if os.path.exists("tests/fixtures/Photos"): 177 | shutil.rmtree("tests/fixtures/Photos") 178 | os.makedirs("tests/fixtures/Photos") 179 | 180 | with mock.patch.object(piexif, "load") as piexif_patched: 181 | piexif_patched.side_effect = InvalidImageDataError 182 | 183 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 184 | # Pass fixed client ID via environment variable 185 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 186 | runner = CliRunner() 187 | result = runner.invoke( 188 | main, 189 | [ 190 | "--username", 191 | "jdoe@gmail.com", 192 | "--password", 193 | "password1", 194 | "--recent", 195 | "1", 196 | "--skip-videos", 197 | "--skip-live-photos", 198 | "--set-exif-datetime", 199 | "--no-progress-bar", 200 | "tests/fixtures/Photos", 201 | ], 202 | ) 203 | print_result_exception(result) 204 | 205 | self.assertIn("DEBUG Looking up all photos...", self._caplog.text) 206 | self.assertIn( 207 | "INFO Downloading the first original photo to tests/fixtures/Photos/ ...", 208 | self._caplog.text, 209 | ) 210 | self.assertIn( 211 | "INFO Downloading tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 212 | self._caplog.text, 213 | ) 214 | self.assertIn( 215 | "DEBUG Error fetching EXIF data for tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 216 | self._caplog.text, 217 | ) 218 | self.assertIn( 219 | "DEBUG Error setting EXIF data for tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 220 | self._caplog.text, 221 | ) 222 | self.assertIn( 223 | "INFO All photos have been downloaded!", self._caplog.text 224 | ) 225 | assert result.exit_code == 0 226 | 227 | def test_skip_existing_downloads(self): 228 | if os.path.exists("tests/fixtures/Photos"): 229 | shutil.rmtree("tests/fixtures/Photos") 230 | os.makedirs("tests/fixtures/Photos/2018/07/31/") 231 | open("tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", "a").close() 232 | open("tests/fixtures/Photos/2018/07/31/IMG_7409.MOV", "a").close() 233 | 234 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 235 | # Pass fixed client ID via environment variable 236 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 237 | runner = CliRunner() 238 | result = runner.invoke( 239 | main, 240 | [ 241 | "--username", 242 | "jdoe@gmail.com", 243 | "--password", 244 | "password1", 245 | "--recent", 246 | "1", 247 | # '--skip-videos', 248 | # "--skip-live-photos", 249 | "--no-progress-bar", 250 | "tests/fixtures/Photos", 251 | ], 252 | ) 253 | print_result_exception(result) 254 | 255 | self.assertIn( 256 | "DEBUG Looking up all photos and videos...", self._caplog.text 257 | ) 258 | self.assertIn( 259 | "INFO Downloading the first original photo or video to tests/fixtures/Photos/ ...", 260 | self._caplog.text, 261 | ) 262 | self.assertIn( 263 | "INFO tests/fixtures/Photos/2018/07/31/IMG_7409.JPG already exists.", 264 | self._caplog.text, 265 | ) 266 | self.assertIn( 267 | "INFO tests/fixtures/Photos/2018/07/31/IMG_7409.MOV already exists.", 268 | self._caplog.text, 269 | ) 270 | self.assertIn( 271 | "INFO All photos have been downloaded!", self._caplog.text 272 | ) 273 | assert result.exit_code == 0 274 | 275 | def test_until_found(self): 276 | base_dir = "tests/fixtures/Photos" 277 | if os.path.exists("tests/fixtures/Photos"): 278 | shutil.rmtree("tests/fixtures/Photos") 279 | os.makedirs("tests/fixtures/Photos/2018/07/30/") 280 | os.makedirs("tests/fixtures/Photos/2018/07/31/") 281 | 282 | files_to_download = [] 283 | files_to_skip = [] 284 | 285 | files_to_download.append(("2018/07/31/IMG_7409.JPG", "photo")) 286 | files_to_download.append(("2018/07/31/IMG_7409-medium.MOV", "photo")) 287 | files_to_skip.append(("2018/07/30/IMG_7408.JPG", "photo")) 288 | files_to_skip.append(("2018/07/30/IMG_7408-medium.MOV", "photo")) 289 | files_to_download.append(("2018/07/30/IMG_7407.JPG", "photo")) 290 | files_to_download.append(("2018/07/30/IMG_7407-medium.MOV", "photo")) 291 | files_to_skip.append(("2018/07/30/IMG_7405.MOV", "video")) 292 | files_to_skip.append(("2018/07/30/IMG_7404.MOV", "video")) 293 | files_to_download.append(("2018/07/30/IMG_7403.MOV", "video")) 294 | files_to_download.append(("2018/07/30/IMG_7402.MOV", "video")) 295 | files_to_skip.append(("2018/07/30/IMG_7401.MOV", "photo")) 296 | files_to_skip.append(("2018/07/30/IMG_7400.JPG", "photo")) 297 | files_to_skip.append(("2018/07/30/IMG_7400-medium.MOV", "photo")) 298 | files_to_skip.append(("2018/07/30/IMG_7399.JPG", "photo")) 299 | files_to_download.append(("2018/07/30/IMG_7399-medium.MOV", "photo")) 300 | 301 | for f in files_to_skip: 302 | open("%s/%s" % (base_dir, f[0]), "a").close() 303 | 304 | with mock.patch("icloudpd.download.download_media") as dp_patched: 305 | dp_patched.return_value = True 306 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 307 | # Pass fixed client ID via environment variable 308 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 309 | runner = CliRunner() 310 | result = runner.invoke( 311 | main, 312 | [ 313 | "--username", 314 | "jdoe@gmail.com", 315 | "--password", 316 | "password1", 317 | "--live-photo-size", 318 | "medium", 319 | "--until-found", 320 | "3", 321 | "--recent", 322 | "20", 323 | "--no-progress-bar", 324 | base_dir, 325 | ], 326 | ) 327 | print_result_exception(result) 328 | 329 | expected_calls = list( 330 | map( 331 | lambda f: call( 332 | ANY, ANY, "%s/%s" % (base_dir, f[0]), 333 | "mediumVideo" if ( 334 | f[1] == 'photo' and f[0].endswith('.MOV') 335 | ) else "original"), 336 | files_to_download, 337 | ) 338 | ) 339 | dp_patched.assert_has_calls(expected_calls) 340 | 341 | self.assertIn( 342 | "DEBUG Looking up all photos and videos...", self._caplog.text 343 | ) 344 | self.assertIn( 345 | "INFO Downloading ??? original photos and videos to tests/fixtures/Photos/ ...", 346 | self._caplog.text, 347 | ) 348 | 349 | for f in files_to_skip: 350 | expected_message = "INFO %s/%s already exists." % (base_dir, f[0]) 351 | self.assertIn(expected_message, self._caplog.text) 352 | 353 | self.assertIn( 354 | "INFO Found 3 consecutive previously downloaded photos. Exiting", 355 | self._caplog.text, 356 | ) 357 | assert result.exit_code == 0 358 | 359 | def test_handle_io_error(self): 360 | if os.path.exists("tests/fixtures/Photos"): 361 | shutil.rmtree("tests/fixtures/Photos") 362 | os.makedirs("tests/fixtures/Photos") 363 | 364 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 365 | # Pass fixed client ID via environment variable 366 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 367 | 368 | with mock.patch("icloudpd.download.open", create=True) as m: 369 | # Raise IOError when we try to write to the destination file 370 | m.side_effect = IOError 371 | 372 | runner = CliRunner() 373 | result = runner.invoke( 374 | main, 375 | [ 376 | "--username", 377 | "jdoe@gmail.com", 378 | "--password", 379 | "password1", 380 | "--recent", 381 | "1", 382 | "--skip-videos", 383 | "--skip-live-photos", 384 | "--no-progress-bar", 385 | "tests/fixtures/Photos", 386 | ], 387 | ) 388 | print_result_exception(result) 389 | 390 | self.assertIn("DEBUG Looking up all photos...", self._caplog.text) 391 | self.assertIn( 392 | "INFO Downloading the first original photo to tests/fixtures/Photos/ ...", 393 | self._caplog.text, 394 | ) 395 | self.assertIn( 396 | "ERROR IOError while writing file to " 397 | "tests/fixtures/Photos/2018/07/31/IMG_7409.JPG! " 398 | "You might have run out of disk space, or the file might " 399 | "be too large for your OS. Skipping this file...", 400 | self._caplog.text, 401 | ) 402 | assert result.exit_code == 0 403 | 404 | def test_handle_session_error_during_download(self): 405 | if os.path.exists("tests/fixtures/Photos"): 406 | shutil.rmtree("tests/fixtures/Photos") 407 | os.makedirs("tests/fixtures/Photos") 408 | 409 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 410 | # Pass fixed client ID via environment variable 411 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 412 | 413 | def mock_raise_response_error(arg): 414 | raise PyiCloudAPIResponseError("Invalid global session", 100) 415 | 416 | with mock.patch("time.sleep") as sleep_mock: 417 | with mock.patch.object(PhotoAsset, "download") as pa_download: 418 | pa_download.side_effect = mock_raise_response_error 419 | 420 | # Let the initial authenticate() call succeed, 421 | # but do nothing on the second try. 422 | orig_authenticate = PyiCloudService.authenticate 423 | 424 | def mocked_authenticate(self): 425 | if not hasattr(self, "already_authenticated"): 426 | orig_authenticate(self) 427 | setattr(self, "already_authenticated", True) 428 | 429 | with mock.patch.object( 430 | PyiCloudService, "authenticate", new=mocked_authenticate 431 | ): 432 | runner = CliRunner() 433 | result = runner.invoke( 434 | main, 435 | [ 436 | "--username", 437 | "jdoe@gmail.com", 438 | "--password", 439 | "password1", 440 | "--recent", 441 | "1", 442 | "--skip-videos", 443 | "--skip-live-photos", 444 | "--no-progress-bar", 445 | "tests/fixtures/Photos", 446 | ], 447 | ) 448 | print_result_exception(result) 449 | 450 | # Error msg should be repeated 5 times 451 | assert ( 452 | self._caplog.text.count( 453 | "Session error, re-authenticating..." 454 | ) 455 | == 5 456 | ) 457 | 458 | self.assertIn( 459 | "INFO Could not download IMG_7409.JPG! Please try again later.", 460 | self._caplog.text, 461 | ) 462 | 463 | # Make sure we only call sleep 4 times (skip the first retry) 464 | self.assertEquals(sleep_mock.call_count, 4) 465 | assert result.exit_code == 0 466 | 467 | def test_handle_session_error_during_photo_iteration(self): 468 | if os.path.exists("tests/fixtures/Photos"): 469 | shutil.rmtree("tests/fixtures/Photos") 470 | os.makedirs("tests/fixtures/Photos") 471 | 472 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 473 | # Pass fixed client ID via environment variable 474 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 475 | 476 | def mock_raise_response_error(offset): 477 | raise PyiCloudAPIResponseError("Invalid global session", 100) 478 | 479 | with mock.patch("time.sleep") as sleep_mock: 480 | with mock.patch.object(PhotoAlbum, "photos_request") as pa_photos_request: 481 | pa_photos_request.side_effect = mock_raise_response_error 482 | 483 | # Let the initial authenticate() call succeed, 484 | # but do nothing on the second try. 485 | orig_authenticate = PyiCloudService.authenticate 486 | 487 | def mocked_authenticate(self): 488 | if not hasattr(self, "already_authenticated"): 489 | orig_authenticate(self) 490 | setattr(self, "already_authenticated", True) 491 | 492 | with mock.patch.object( 493 | PyiCloudService, "authenticate", new=mocked_authenticate 494 | ): 495 | runner = CliRunner() 496 | result = runner.invoke( 497 | main, 498 | [ 499 | "--username", 500 | "jdoe@gmail.com", 501 | "--password", 502 | "password1", 503 | "--recent", 504 | "1", 505 | "--skip-videos", 506 | "--skip-live-photos", 507 | "--no-progress-bar", 508 | "tests/fixtures/Photos", 509 | ], 510 | ) 511 | print_result_exception(result) 512 | 513 | # Error msg should be repeated 5 times 514 | assert ( 515 | self._caplog.text.count( 516 | "Session error, re-authenticating..." 517 | ) 518 | == 5 519 | ) 520 | 521 | self.assertIn( 522 | "INFO iCloud re-authentication failed! Please try again later.", 523 | self._caplog.text, 524 | ) 525 | # Make sure we only call sleep 4 times (skip the first retry) 526 | self.assertEquals(sleep_mock.call_count, 4) 527 | 528 | assert result.exit_code == -1 529 | 530 | def test_handle_connection_error(self): 531 | if os.path.exists("tests/fixtures/Photos"): 532 | shutil.rmtree("tests/fixtures/Photos") 533 | os.makedirs("tests/fixtures/Photos") 534 | 535 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 536 | # Pass fixed client ID via environment variable 537 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 538 | 539 | def mock_raise_response_error(arg): 540 | raise ConnectionError("Connection Error") 541 | 542 | with mock.patch.object(PhotoAsset, "download") as pa_download: 543 | pa_download.side_effect = mock_raise_response_error 544 | 545 | # Let the initial authenticate() call succeed, 546 | # but do nothing on the second try. 547 | orig_authenticate = PyiCloudService.authenticate 548 | 549 | def mocked_authenticate(self): 550 | if not hasattr(self, "already_authenticated"): 551 | orig_authenticate(self) 552 | setattr(self, "already_authenticated", True) 553 | 554 | with mock.patch("icloudpd.constants.WAIT_SECONDS", 0): 555 | with mock.patch.object( 556 | PyiCloudService, "authenticate", new=mocked_authenticate 557 | ): 558 | runner = CliRunner() 559 | result = runner.invoke( 560 | main, 561 | [ 562 | "--username", 563 | "jdoe@gmail.com", 564 | "--password", 565 | "password1", 566 | "--recent", 567 | "1", 568 | "--skip-videos", 569 | "--skip-live-photos", 570 | "--no-progress-bar", 571 | "tests/fixtures/Photos", 572 | ], 573 | ) 574 | print_result_exception(result) 575 | 576 | # Error msg should be repeated 5 times 577 | assert ( 578 | self._caplog.text.count( 579 | "Error downloading IMG_7409.JPG, retrying after 0 seconds..." 580 | ) 581 | == 5 582 | ) 583 | 584 | self.assertIn( 585 | "INFO Could not download IMG_7409.JPG! Please try again later.", 586 | self._caplog.text, 587 | ) 588 | assert result.exit_code == 0 589 | 590 | def test_missing_size(self): 591 | base_dir = "tests/fixtures/Photos" 592 | if os.path.exists("tests/fixtures/Photos"): 593 | shutil.rmtree("tests/fixtures/Photos") 594 | os.makedirs("tests/fixtures/Photos") 595 | 596 | with mock.patch.object(PhotoAsset, "download") as pa_download: 597 | pa_download.return_value = False 598 | 599 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 600 | # Pass fixed client ID via environment variable 601 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 602 | runner = CliRunner() 603 | result = runner.invoke( 604 | main, 605 | [ 606 | "--username", 607 | "jdoe@gmail.com", 608 | "--password", 609 | "password1", 610 | "--recent", 611 | "3", 612 | "--no-progress-bar", 613 | base_dir, 614 | ], 615 | ) 616 | print_result_exception(result) 617 | 618 | self.assertIn( 619 | "DEBUG Looking up all photos and videos...", self._caplog.text 620 | ) 621 | self.assertIn( 622 | "INFO Downloading 3 original photos and videos to tests/fixtures/Photos/ ...", 623 | self._caplog.text, 624 | ) 625 | 626 | # These error messages should not be repeated more than once 627 | assert ( 628 | self._caplog.text.count( 629 | "ERROR Could not find URL to download IMG_7409.JPG for size original!" 630 | ) 631 | == 1 632 | ) 633 | assert ( 634 | self._caplog.text.count( 635 | "ERROR Could not find URL to download IMG_7408.JPG for size original!" 636 | ) 637 | == 1 638 | ) 639 | assert ( 640 | self._caplog.text.count( 641 | "ERROR Could not find URL to download IMG_7407.JPG for size original!" 642 | ) 643 | == 1 644 | ) 645 | 646 | self.assertIn( 647 | "INFO All photos have been downloaded!", self._caplog.text 648 | ) 649 | assert result.exit_code == 0 650 | 651 | def test_size_fallback_to_original(self): 652 | base_dir = "tests/fixtures/Photos" 653 | if os.path.exists("tests/fixtures/Photos"): 654 | shutil.rmtree("tests/fixtures/Photos") 655 | os.makedirs("tests/fixtures/Photos") 656 | 657 | with mock.patch("icloudpd.download.download_media") as dp_patched: 658 | dp_patched.return_value = True 659 | 660 | with mock.patch.object(PhotoAsset, "versions") as pa: 661 | pa.return_value = ["original", "medium"] 662 | 663 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 664 | # Pass fixed client ID via environment variable 665 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 666 | runner = CliRunner() 667 | result = runner.invoke( 668 | main, 669 | [ 670 | "--username", 671 | "jdoe@gmail.com", 672 | "--password", 673 | "password1", 674 | "--recent", 675 | "1", 676 | "--size", 677 | "thumb", 678 | "--no-progress-bar", 679 | base_dir, 680 | ], 681 | ) 682 | print_result_exception(result) 683 | self.assertIn( 684 | "DEBUG Looking up all photos and videos...", 685 | self._caplog.text, 686 | ) 687 | self.assertIn( 688 | "INFO Downloading the first thumb photo or video to tests/fixtures/Photos/ ...", 689 | self._caplog.text, 690 | ) 691 | self.assertIn( 692 | "INFO Downloading tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 693 | self._caplog.text, 694 | ) 695 | self.assertIn( 696 | "INFO All photos have been downloaded!", self._caplog.text 697 | ) 698 | dp_patched.assert_called_once_with( 699 | ANY, 700 | ANY, 701 | "tests/fixtures/Photos/2018/07/31/IMG_7409.JPG", 702 | "original", 703 | ) 704 | 705 | assert result.exit_code == 0 706 | 707 | def test_force_size(self): 708 | base_dir = "tests/fixtures/Photos" 709 | if os.path.exists("tests/fixtures/Photos"): 710 | shutil.rmtree("tests/fixtures/Photos") 711 | os.makedirs("tests/fixtures/Photos") 712 | 713 | with mock.patch("icloudpd.download.download_media") as dp_patched: 714 | dp_patched.return_value = True 715 | 716 | with mock.patch.object(PhotoAsset, "versions") as pa: 717 | pa.return_value = ["original", "medium"] 718 | 719 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 720 | # Pass fixed client ID via environment variable 721 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 722 | runner = CliRunner() 723 | result = runner.invoke( 724 | main, 725 | [ 726 | "--username", 727 | "jdoe@gmail.com", 728 | "--password", 729 | "password1", 730 | "--recent", 731 | "1", 732 | "--size", 733 | "thumb", 734 | "--force-size", 735 | "--no-progress-bar", 736 | base_dir, 737 | ], 738 | ) 739 | print_result_exception(result) 740 | 741 | self.assertIn( 742 | "DEBUG Looking up all photos and videos...", 743 | self._caplog.text, 744 | ) 745 | self.assertIn( 746 | "INFO Downloading the first thumb photo or video to tests/fixtures/Photos/ ...", 747 | self._caplog.text, 748 | ) 749 | self.assertIn( 750 | "ERROR thumb size does not exist for IMG_7409.JPG. Skipping...", 751 | self._caplog.text, 752 | ) 753 | self.assertIn( 754 | "INFO All photos have been downloaded!", self._caplog.text 755 | ) 756 | dp_patched.assert_not_called 757 | 758 | assert result.exit_code == 0 759 | 760 | 761 | def test_invalid_creation_date(self): 762 | base_dir = "tests/fixtures/Photos" 763 | if os.path.exists("tests/fixtures/Photos"): 764 | shutil.rmtree("tests/fixtures/Photos") 765 | os.makedirs("tests/fixtures/Photos") 766 | 767 | with mock.patch.object(PhotoAsset, "created", new_callable=mock.PropertyMock) as dt_mock: 768 | # Can't mock `astimezone` because it's a readonly property, so have to 769 | # create a new class that inherits from datetime.datetime 770 | class NewDateTime(datetime.datetime): 771 | def astimezone(self, tz=None): 772 | raise ValueError('Invalid date') 773 | dt_mock.return_value = NewDateTime(2018,1,1,0,0,0) 774 | 775 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 776 | # Pass fixed client ID via environment variable 777 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 778 | runner = CliRunner() 779 | result = runner.invoke( 780 | main, 781 | [ 782 | "--username", 783 | "jdoe@gmail.com", 784 | "--password", 785 | "password1", 786 | "--recent", 787 | "1", 788 | "--skip-live-photos", 789 | "--no-progress-bar", 790 | base_dir, 791 | ], 792 | ) 793 | print_result_exception(result) 794 | 795 | self.assertIn( 796 | "DEBUG Looking up all photos and videos...", 797 | self._caplog.text, 798 | ) 799 | self.assertIn( 800 | "INFO Downloading the first original photo or video to tests/fixtures/Photos/ ...", 801 | self._caplog.text, 802 | ) 803 | self.assertIn( 804 | "ERROR Could not convert photo created date to local timezone (2018-01-01 00:00:00)", 805 | self._caplog.text, 806 | ) 807 | self.assertIn( 808 | "INFO Downloading tests/fixtures/Photos/2018/01/01/IMG_7409.JPG", 809 | self._caplog.text, 810 | ) 811 | self.assertIn( 812 | "INFO All photos have been downloaded!", self._caplog.text 813 | ) 814 | assert result.exit_code == 0 815 | 816 | def test_invalid_creation_year(self): 817 | base_dir = "tests/fixtures/Photos" 818 | if os.path.exists("tests/fixtures/Photos"): 819 | shutil.rmtree("tests/fixtures/Photos") 820 | os.makedirs("tests/fixtures/Photos") 821 | 822 | with mock.patch.object(PhotoAsset, "created", new_callable=mock.PropertyMock) as dt_mock: 823 | # Can't mock `astimezone` because it's a readonly property, so have to 824 | # create a new class that inherits from datetime.datetime 825 | class NewDateTime(datetime.datetime): 826 | def astimezone(self, tz=None): 827 | raise ValueError('Invalid date') 828 | dt_mock.return_value = NewDateTime(5,1,1,0,0,0) 829 | 830 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 831 | # Pass fixed client ID via environment variable 832 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 833 | runner = CliRunner() 834 | result = runner.invoke( 835 | main, 836 | [ 837 | "--username", 838 | "jdoe@gmail.com", 839 | "--password", 840 | "password1", 841 | "--recent", 842 | "1", 843 | "--skip-live-photos", 844 | "--no-progress-bar", 845 | base_dir, 846 | ], 847 | ) 848 | print_result_exception(result) 849 | 850 | self.assertIn( 851 | "DEBUG Looking up all photos and videos...", 852 | self._caplog.text, 853 | ) 854 | self.assertIn( 855 | "INFO Downloading the first original photo or video to tests/fixtures/Photos/ ...", 856 | self._caplog.text, 857 | ) 858 | if sys.version_info[0] < 3: 859 | # Python 2.7 860 | self.assertIn( 861 | "ERROR Photo created date was not valid (0005-01-01 00:00:00)", 862 | self._caplog.text, 863 | ) 864 | self.assertIn( 865 | "INFO Downloading tests/fixtures/Photos/1970/01/01/IMG_7409.JPG", 866 | self._caplog.text, 867 | ) 868 | else: 869 | self.assertIn( 870 | "ERROR Could not convert photo created date to local timezone (0005-01-01 00:00:00)", 871 | self._caplog.text, 872 | ) 873 | if sys.version_info[1] >= 7: 874 | # Python 3.7 875 | self.assertIn( 876 | "INFO Downloading tests/fixtures/Photos/0005/01/01/IMG_7409.JPG", 877 | self._caplog.text, 878 | ) 879 | else: 880 | # Python 3.6 881 | self.assertIn( 882 | "INFO Downloading tests/fixtures/Photos/5/01/01/IMG_7409.JPG", 883 | self._caplog.text, 884 | ) 885 | self.assertIn( 886 | "INFO All photos have been downloaded!", self._caplog.text 887 | ) 888 | assert result.exit_code == 0 889 | 890 | 891 | def test_unknown_item_type(self): 892 | base_dir = "tests/fixtures/Photos" 893 | if os.path.exists("tests/fixtures/Photos"): 894 | shutil.rmtree("tests/fixtures/Photos") 895 | os.makedirs("tests/fixtures/Photos") 896 | 897 | with mock.patch("icloudpd.download.download_media") as dp_patched: 898 | dp_patched.return_value = True 899 | 900 | with mock.patch.object(PhotoAsset, "item_type", new_callable=mock.PropertyMock) as it_mock: 901 | it_mock.return_value = 'unknown' 902 | 903 | with vcr.use_cassette("tests/vcr_cassettes/listing_photos.yml"): 904 | # Pass fixed client ID via environment variable 905 | os.environ["CLIENT_ID"] = "DE309E26-942E-11E8-92F5-14109FE0B321" 906 | runner = CliRunner() 907 | result = runner.invoke( 908 | main, 909 | [ 910 | "--username", 911 | "jdoe@gmail.com", 912 | "--password", 913 | "password1", 914 | "--recent", 915 | "1", 916 | "--no-progress-bar", 917 | base_dir, 918 | ], 919 | ) 920 | print_result_exception(result) 921 | 922 | self.assertIn( 923 | "DEBUG Looking up all photos and videos...", 924 | self._caplog.text, 925 | ) 926 | self.assertIn( 927 | "INFO Downloading the first original photo or video to tests/fixtures/Photos/ ...", 928 | self._caplog.text, 929 | ) 930 | self.assertIn( 931 | "INFO Skipping IMG_7409.JPG, only downloading photos and videos. (Item type was: unknown)", 932 | self._caplog.text, 933 | ) 934 | self.assertIn( 935 | "INFO All photos have been downloaded!", self._caplog.text 936 | ) 937 | dp_patched.assert_not_called 938 | 939 | assert result.exit_code == 0 940 | --------------------------------------------------------------------------------