├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── conftest.py ├── doc ├── LICENSE └── changelog.md ├── logs └── empty_file.txt ├── lycheesync ├── __init__.py ├── lycheedao.py ├── lycheemodel.py ├── lycheesyncer.py ├── sync.py ├── update_scripts │ ├── __init__.py │ └── inf_to_lychee_2_6_2.py └── utils │ ├── __init__.py │ ├── boilerplatecode.py │ └── configuration.py ├── main.py ├── requirements.txt ├── ressources ├── conf.json ├── logging.json ├── lychee.sql ├── pytest.ini └── test_conf.json ├── setup.py └── tests ├── __init__.py ├── configuration.py ├── pics ├── FußÄ-Füße │ ├── Füße.jpg │ └── ok_ßüöä.jpg ├── aaa │ ├── Lychees---Nature_s-Pride.jpg │ ├── Watercolor_Lychee.jpg │ ├── fruit-lychee.jpg │ └── lychee-fruit-21262197.jpg ├── album1 │ └── large.1.jpg ├── album2 │ ├── album21 │ │ └── 6640926-single-lychee-also-known-as-chinese-gooseberry-isolated-against-white-background.jpg │ ├── album22 │ │ └── 14828607-lychee.jpg │ ├── lychee_heart_1_by_yuri_chan1018-d4176jl.jpg │ └── one-cut-lychee.jpg ├── album3 │ ├── Lychees---Nature_s-Pride.jpg │ ├── Watercolor_Lychee.jpg │ ├── fruit-lychee.jpg │ └── lychee-fruit-21262197.jpg ├── corrupted_file │ └── Lychees---Nature_s-Pride.jpg ├── duplicates │ ├── large.1.jpg │ └── large.2.jpg ├── empty_album │ └── tocommit.txt ├── invalid_takedate │ └── large.1.jpg ├── invalid_taketime │ └── IMG_0205.JPG ├── mini │ └── P1060266.JPG ├── real_date │ ├── fruit-lychee.jpg │ └── fruit-lychee2.jpg ├── rotation │ ├── P1010335.JPG │ ├── P1010336.JPG │ ├── P1010337.JPG │ └── P1010338.JPG ├── with'"quotes │ └── iu'"`stanragename.jpg └── zzzz │ ├── Watercolor_Lychee.jpg │ ├── fruit-lychee.jpg │ └── lychee-fruit-21262197.jpg ├── standalone ├── __init__.py ├── db_test.py └── epoch_test.py ├── test_main.py └── testutils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | 37 | tmptest/ 38 | venv3.4/* 39 | venv2.7/* 40 | test/* 41 | logs/* 42 | exiftest.py 43 | myconf.json 44 | testconf.json 45 | profile.py 46 | .cache 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | branches: 4 | only: 5 | - master 6 | - dev 7 | 8 | python: 9 | - "2.7" 10 | - "3.4" 11 | - "3.5" 12 | 13 | # command to install dependencies 14 | install: 15 | - pip install -r requirements.txt 16 | - pip freeze 17 | - pip install coveralls 18 | 19 | services: 20 | - mysql 21 | 22 | before_install: 23 | - sudo apt-get install python-dev python3-dev 24 | 25 | before_script: 26 | - mysql -u root -e 'create database lychee_ci DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' 27 | - mysql -u root -e "create user 'lychee_ci'@'localhost' IDENTIFIED BY 'lychee_ci';" 28 | - mysql -u root -e "GRANT ALL ON lychee_ci.* TO 'lychee_ci'@'localhost';" 29 | 30 | # command to run tests 31 | script: 32 | coverage run -m --source ./lycheesync py.test -c ./ressources/pytest.ini --showlocals --duration=3 -v -s --confpath=${PWD}/ressources/test_conf.json 33 | 34 | after_success: 35 | - coveralls 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | test_all: 4 | python main.py ./tmptest/ /var/www/Lychee/ ./ressources/test_conf.json -v -d 5 | 6 | test: 7 | py.test -c ./ressources/pytest.ini --cov=lycheesync --pep8 --showlocals --duration=3 -v -s --confpath=${PWD}/ressources/test_conf.json --cov-report term-missing 8 | # py.test -c ./ressources/pytest.ini --showlocals --duration=3 -v -s --confpath=${PWD}/ressources/test_conf.json 9 | 10 | testdev: 11 | py.test -c ./ressources/pytest.ini --cov=lycheesync --pep8 --showlocals --duration=3 -v -s --confpath=${PWD}/ressources/test_conf.json -k test_visually_check_logs 12 | 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lycheesync 2 | 3 | [![Build Status](https://travis-ci.org/GustavePate/lycheesync.svg)](https://travis-ci.org/GustavePate/lycheesync) [![Coverage Status](https://img.shields.io/coveralls/GustavePate/lycheesync/master.svg)](https://coveralls.io/github/GustavePate/lycheesync?branch=master) 4 | 5 | Lycheesync is a command line tool to synchronise a directory containing photos with Lychee. 6 | * Lycheesync is meant to be used on the same server that run Lychee. If your photo source directory is on another computer, use synchronize tools like rsync or owncloud. 7 | * Lycheesync is often meant to be run regulary and automatically, use cron for this (or monitor [filesystem events](https://github.com/seb-m/pyinotify) if you want your photos really fast online) 8 | 9 | ## WARNING: Breaking changes 10 | 11 | Sorry for the inconvenience but Lycheesync has change a lot in the last weeks. 12 | I added a few dependencies and remove others. 13 | As an exemple the mysql driver has changed, so... 14 | Check the install Chapter ! 15 | 16 | PS: I strongly recommand to use python3.4 with a virtualenv even if python2.7 will still be supported in the following months. 17 | 18 | # Issue / logs 19 | 20 | If you have read the documentation below and it still doesn't work as expected for you, feel free to submit a github issue. 21 | 22 | Complete logs for the last run can be found in `logs/lycheesync.log`, if it's relevant, please attach them to your issue. 23 | 24 | ## Context 25 | 26 | This project was created to synchronize an [owncloud](http://owncloud.org/) photo repositories and [Lychee](http://lychee.electerious.com/). 27 | It turns out it can, totally or partially, enslave Lychee with any given directory structure. 28 | 29 | The program is simple it scans a directory for files and subdirectories: 30 | - subdirectories are converted to Lychee albums 31 | - files are imported in Lychee as photos 32 | 33 | You can choose between 3 behaviours: 34 | - **Lychee as a slave**: Lychee db is drop before each run `-d option` 35 | - **Lychee as a slave only for album in the source directories**: albums existing in 36 | Lychee but not in the source directory will be kept `-r option` 37 | - **Keep existing Lychee albums and photos** The program will try to know if a photo in the 38 | source directory has already been imported in Lychee and does nothing in this case, this is the default behaviour 39 | 40 | 41 | ## What's new 42 | 43 | See [changelog](./doc/changelog.md) 44 | 45 | ## Install 46 | 47 | ### Retrieve the project 48 | 49 | `git clone https://github.com/GustavePate/lycheesync` 50 | 51 | ### Install dependencies 52 | 53 | Then you have to install the following dependencies: 54 | 55 | - python 2.7, 3.4 or 3.5 (python 3 prefered !) 56 | - pillow 57 | - dateutils 58 | - [pymysql](https://github.com/PyMySQL/PyMySQL) 59 | - [click](http://click.pocoo.org/) 60 | 61 | 62 | #### Using a virtual env (the GOOD way) 63 | 64 | On debian based Linux 65 | 66 | sudo apt-get install python3-dev python3.4-venv libjpeg-dev zlib1g-dev 67 | cd /path/to/lycheesync 68 | pyvenv-3.4 ./venv3.4 69 | . ./venv3.4/bin/activate 70 | which pip # should give you a path in your newly created ./venv3.4 dir 71 | # if not execute: curl https://raw.githubusercontent.com/pypa/pip/master/contrib/get-pip.py | python 72 | pip install -r requirements.txt 73 | 74 | And wait for compilation to finish ;) 75 | 76 | #### Using the distro package manager (the BAD way) 77 | 78 | On debian based Linux 79 | 80 | sudo apt-get install python3-dev python3 python3-pymysql python3-click python3-pil python3-dateutil libjpeg-dev 81 | 82 | PS: Depending on your distro version, you may need to activate universe repository for you distribution first. 83 | 84 | ### Adjust configuration 85 | 86 | Finally, adjust the `ressources/conf.json` file to you use case. 87 | Explanations in next chapter. 88 | 89 | ## Basic usage 90 | 91 | ### Configuration 92 | 93 | The configuration file is straight-forward. 94 | Simply enter your Lychee DB configuration. 95 | publicAlbum should be set to 1 if you want to make public all your photos. 96 | excludeAlbums is a list of patterns which albums not to include. The example below will not import the album `/home/snowden/albums/secret` and any directory named `.DAV` or `temp`: 97 | 98 | 99 | ```json 100 | { 101 | "db":"lychee", 102 | "dbUser":"lychee", 103 | "dbPassword":"cheely", 104 | "dbHost":"localhost", 105 | "thumbQuality":80, 106 | "publicAlbum": 0, 107 | "excludeAlbums": [ 108 | "/home/snowden/albums/secret", 109 | "*/.DAV", 110 | "*/temp" 111 | ] 112 | } 113 | ``` 114 | 115 | ### Command line parameters 116 | 117 | The basic usage is `python -m lycheesync.sync srcdir lycheepath conf` 118 | 119 | Where: 120 | - `srcdir` is the directory containing photos you want to add to Lychee 121 | - `lycheepath` is the path were you installed Lychee (usually `/var/www/lychee`) 122 | - `conf` is the full path to your configuration file (usually `./ressources/conf.json`) 123 | 124 | ### Explanation 125 | 126 | The default mod is a **merge** mode. 127 | 128 | Given the following source tree: 129 | 130 | ```text 131 | _srcdir 132 | |_album1 133 | |_a1p1.jpg 134 | |_a1p2.jpg 135 | |_album2 136 | |_album21 137 | |_a21p1.jpg 138 | |_a21p2.jpg 139 | |_album22 140 | |_a22p1.jpg 141 | ``` 142 | 143 | And this lychee prexisting structure: 144 | 145 | ```text 146 | |_album1 147 | |_a1p1.jpg 148 | |_a1p3.jpg 149 | |_album3 150 | |_a3p1.jpg 151 | ``` 152 | 153 | Lychee doesn't support yet sub-albums so any sub-directory in your source directory will be flat-out 154 | The resulting lychee structure will be: 155 | 156 | 157 | ```text 158 | |_album1 159 | |_a1p1.jpg (won't be re-imported by default) 160 | |_a1p2.jpg 161 | |_a1p3.jpg 162 | |_album2_album21 (notice directory / subdirectory concatenation) 163 | |_a21p1.jpg 164 | |_a21p2.jpg 165 | |_album2_album22 166 | |_a22p1.jpg 167 | |_album3 168 | |_a3p1.jpg 169 | ``` 170 | 171 | 172 | ### Counters 173 | 174 | At the end of the script a few counters will be displayed in order to keep you informed of what have been done. 175 | 176 | ```text 177 | Directory scanned: /var/www/lychee/Lychee/dirsync/test/ 178 | Created albums: 4 179 | 10 photos imported on 10 discovered 180 | ``` 181 | 182 | ## Advanced usage 183 | 184 | ### Command line switches 185 | 186 | You can choose between the following options to adjust the program behaviour: 187 | 188 | - `-v` **verbose mode**. A little more output 189 | - `-r` **replace album mode**. If a pre-existing album is found in Lychee that match a soon to be imported album. The pre-existing album is removed before hand. Usefull if you want to have lychee in slave mode only for a few albums 190 | - `-d` **drop all mode**. Everything in Lychee is dropped before import. Usefull to make lychee a slave of another repository 191 | - `-l` **link mode**. Don't copy files from source folder to lychee directory structure, just create symbolic links (thumbnails will however be created in lychee's directory structure) 192 | - `-s` **sort mode**. Sort album by name in lychee. Could be usefull if your album names start with the date (YYYYMMDD). 193 | - `-c` `--sanitycheck` **sanity check mode**. Will remove empty album, orphan files, broken links... 194 | 195 | 196 | ### Choose your album cover 197 | 198 | Add `_star` at the end of one filename in a directory and this photo will be stared, making it your album cover. Ex: `P1000274_star.JPG` 199 | 200 | ### Using crontab to automate synchronization 201 | 202 | Add this line in your crontab (`crontab -e`) to synchronize a photo directory to your lychee installation every day at 2 am. 203 | 204 | 0 2 * * * cd /path/to/lycheesync && . ./venv3.4/bin/activate && python -m lycheesync.sync /path/to/photo_directory/ /var/www/path/to/lychee/ ./ressources/conf.json -d > /tmp/lycheesync.out.log 2>&1 205 | 206 | 207 | ## Technical doc 208 | 209 | This code is pep8 compliant and well documented, if you want to contribute, thanks to 210 | keep it this way. 211 | 212 | This project files are: 213 | * lycheesync/sync.py: argument parsing and conf reading, defer work to lycheesyncer 214 | * lycheesync/lycheesyncer: logic and filesystem operations 215 | * lycheesync/lycheedao: database operations 216 | * lycheesync/lycheemodel: a lychee photo representation, manage exif tag parsing too 217 | * ressources/conf.json: the configuration file 218 | 219 | 220 | # Licence 221 | 222 | [MIT License](./doc/LICENSE) 223 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | import json 5 | import os 6 | import logging 7 | from tests.configuration import TestBorg 8 | from tests.testutils import TestUtils 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | # py.test tweaking 13 | # in this exemple: initialize configuration borg for the whole test session 14 | 15 | 16 | def pytest_report_header(config): 17 | """ return a string in test report header """ 18 | return "Hey this are the tests" 19 | 20 | 21 | def pytest_addoption(parser): 22 | """create a confpath command line arg for py.test""" 23 | parser.addoption( 24 | '--confpath', 25 | dest="confpath", 26 | action="append", 27 | default=[], 28 | help="configuration full path") 29 | 30 | 31 | @pytest.fixture(scope="function") 32 | def clean(request): 33 | """ will be run for each test function see pytest.ini """ 34 | tu = TestUtils() 35 | if tu.db_exists(): 36 | tu.clean_db() 37 | tu.clean_fs() 38 | 39 | 40 | @pytest.fixture(scope="function") 41 | def carriagereturn(request): 42 | """ will be run for each test function see pytest.ini """ 43 | print(" ") 44 | 45 | 46 | @pytest.fixture(scope="session") 47 | def initdb_and_fs(request): 48 | pass 49 | #  print("#FIXTURE: init db and fs") 50 | # tu = TestUtils() 51 | # Impossible because conf not loaded 52 | # tu.make_fake_lychee_db() 53 | # tu.drop_db() 54 | # tu.make_fake_lychee_fs(tu.conf['lycheepath']) 55 | 56 | 57 | @pytest.fixture(scope="session") 58 | def initloggers(request): 59 | print("#FIXTURE: initloggers") 60 | """ will be run for each test session see pytest.ini """ 61 | # initialize basic loggers 62 | print("****** INIT LOGGERS ******") 63 | logging.basicConfig( 64 | level=logging.DEBUG, 65 | format='%(asctime)s.%(msecs)03d %(levelname)s - %(name)s.%(funcName)s l. %(lineno)d - %(message)s', 66 | datefmt='%H:%M:%S') 67 | 68 | 69 | @pytest.fixture(scope="session") 70 | def confborg(request): 71 | """ 72 | - allow confborg to be a valid test parameter, 73 | this code will be executed before the test code 74 | once per session 75 | """ 76 | print("#FIXTURE: confborg") 77 | 78 | def run_only_at_session_end(): 79 | print("\n ********** End of test session **********") 80 | request.addfinalizer(run_only_at_session_end) 81 | 82 | print("********** Test session init **********") 83 | 84 | if pytest.config.getoption('confpath') is not None: 85 | print("Launch test with conf:", pytest.config.getoption('confpath', default=None)) 86 | conf_path = pytest.config.getoption('confpath') 87 | # previously returns a list, get the string 88 | if len(pytest.config.getoption('confpath')) > 0: 89 | conf_path = pytest.config.getoption('confpath')[0] 90 | 91 | if os.path.exists(conf_path): 92 | with open(conf_path, 'rt') as f: 93 | conf = json.load(f) 94 | 95 | # add data path which is calculated at run time 96 | conf['data_path'] = os.path.dirname(os.path.dirname(conf_path)) 97 | conf['data_path'] = os.path.join(conf['data_path'], 'data') 98 | res = TestBorg(conf) 99 | logger.info("TestBorg initialized") 100 | return res 101 | -------------------------------------------------------------------------------- /doc/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Gustave Paté 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /doc/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v3.0.9 4 | 5 | *Warning* this is a breaking release new python packages must be installed (see the Install section in ReadMe) 6 | 7 | - Tested against Lychee 3.0.9 8 | - automated testing 9 | - travis ci 10 | - compatibility with python 3 11 | - don't lower case album name 12 | - photos ordered by name in an album 13 | - make use of the DateTimeOriginal exif tag 14 | - dependency change: switch to a pure python mysql driver [pymysql](https://github.com/PyMySQL/PyMySQL) 15 | - dependency change: use [click](http://click.pocoo.org/) to parse arguments 16 | - new dependency: [pexif](https://github.com/hMatoba/Piexif) Thanks to @hMatoba for his work ! 17 | 18 | ## v3.0.1 19 | - change versioning to match lychee's 20 | 21 | 22 | ## v2.7.1 23 | - change versioning to match lychee's 24 | - don't import duplicated files 25 | - handle corrupted files correctly 26 | - unicode directory name supported 27 | 28 | ## v2.6 29 | - change versioning to match lychee's 30 | - lychee 2.6 support 31 | - fixed some permission problem: give the photo files the same group and owner than lychee uploads directory + rwx permission for group and user 32 | - added an *update mode* to fix problem experienced by those who used lycheesync with a lychee 2.6 before this version. To update your lychee db an photo repository, just add the -u switch to your usual call (you can run it anyway, it won't break anything:) 33 | 34 | `python main.py srcdir lycheepath conf -u` 35 | 36 | ## v1.3 37 | - lychee 2.5 support 38 | 39 | 40 | ## v1.2 41 | - added exif orientation support which was not totally fixed in luchee 2.0 ;) 42 | - a photo containing 'star' or 'cover' in its filename will be automatically starred, thus making it the album cover 43 | - photo titles now equals original filenames 44 | 45 | ## v1.1 46 | - added suport for lychee 2.1 47 | - removed exif orientation support (fixed in lychee 2.0) 48 | - added takedate and taketime in photo description in order to be able to use the sort by description functionality of lychee 2.0 49 | - albums display order is "sorted by name" 50 | - album date is now the max takedate/taketime of its photos if exif data exists (if no, import date is used) 51 | 52 | ## v1.0 53 | - initial version 54 | 55 | -------------------------------------------------------------------------------- /logs/empty_file.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/logs/empty_file.txt -------------------------------------------------------------------------------- /lycheesync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/lycheesync/__init__.py -------------------------------------------------------------------------------- /lycheesync/lycheedao.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | import pymysql 6 | import datetime 7 | import re 8 | import logging 9 | import time 10 | import random 11 | from dateutil.parser import parse 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class LycheeDAO: 17 | 18 | """ 19 | Implements linking with Lychee DB 20 | """ 21 | 22 | db = None 23 | db2 = None 24 | conf = None 25 | albumslist = {} 26 | 27 | def __init__(self, conf): 28 | """ 29 | Takes a dictionnary of conf as input 30 | """ 31 | try: 32 | self.conf = conf 33 | if 'dbSocket' in self.conf: 34 | logger.debug("Connection to db in SOCKET mode") 35 | logger.error("host: %s", self.conf['dbHost']) 36 | logger.error("user: %s", self.conf['dbUser']) 37 | logger.error("password: %s", self.conf['dbPassword']) 38 | logger.error("db: %s", self.conf['db']) 39 | logger.error("unix_socket: %s", self.conf['dbSocket']) 40 | self.db = pymysql.connect(host=self.conf['dbHost'], 41 | user=self.conf['dbUser'], 42 | passwd=self.conf['dbPassword'], 43 | db=self.conf['db'], 44 | charset='utf8mb4', 45 | unix_socket=self.conf['dbSocket'], 46 | cursorclass=pymysql.cursors.DictCursor) 47 | else: 48 | logger.debug("Connection to db in NO SOCKET mode") 49 | self.db = pymysql.connect(host=self.conf['dbHost'], 50 | user=self.conf['dbUser'], 51 | passwd=self.conf['dbPassword'], 52 | db=self.conf['db'], 53 | charset='utf8mb4', 54 | cursorclass=pymysql.cursors.DictCursor) 55 | 56 | cur = self.db.cursor() 57 | cur.execute("set names utf8;") 58 | 59 | if self.conf["dropdb"]: 60 | self.dropAll() 61 | 62 | self.loadAlbumList() 63 | 64 | except Exception as e: 65 | logger.error(e) 66 | raise 67 | 68 | def getUniqPhotoId(self): 69 | id = self.getUniqTimeBasedId() 70 | nbtry = 1 71 | while (self.photoIdExists(id)): 72 | id = self.getUniqTimeBasedId() 73 | nbtry += 1 74 | if (nbtry == 5): 75 | raise Exception("didn't manage to create uniq id") 76 | return id 77 | 78 | def getUniqAlbumId(self): 79 | id = self.getUniqTimeBasedId() 80 | nbtry = 1 81 | while (self.albumIdExists(id)): 82 | id = self.getUniqTimeBasedId() 83 | nbtry += 1 84 | if (nbtry == 5): 85 | raise Exception("didn't manage to create uniq id") 86 | return id 87 | 88 | def getUniqTimeBasedId(self): 89 | # Compute Photo ID 90 | id = str(int(time.time())) 91 | # not precise enough 92 | length = len(id) 93 | if length < 14: 94 | missing_char = 14 - length 95 | r = random.random() 96 | r = str(r) 97 | # last missing_char char 98 | filler = r[-missing_char:] 99 | id = id + filler 100 | return id 101 | 102 | def getAlbumNameDBWidth(self): 103 | res = 50 # default value 104 | query = "show columns from lychee_albums where Field='title'" 105 | cur = self.db.cursor() 106 | try: 107 | cur.execute(query) 108 | row = cur.fetchone() 109 | type = row['Type'] 110 | # is type ok 111 | p = re.compile('varchar\(\d+\)', re.IGNORECASE) 112 | if p.match(type): 113 | # remove varchar(and) 114 | p = re.compile('\d+', re.IGNORECASE) 115 | ints = p.findall(type) 116 | if len(ints) > 0: 117 | res = int(ints[0]) 118 | else: 119 | logger.warn("getAlbumNameDBWidth unable to find album name width fallback to default") 120 | except Exception as e: 121 | logger.exception(e) 122 | logger.warn("getAlbumNameDBWidth while executing: " + cur._last_executed) 123 | finally: 124 | return res 125 | 126 | def getAlbumMinMaxIds(self): 127 | """ 128 | returns min, max album ids 129 | """ 130 | min_album_query = "select min(id) as min from lychee_albums" 131 | max_album_query = "select max(id) as max from lychee_albums" 132 | try: 133 | min = -1 134 | max = -1 135 | cur = self.db.cursor() 136 | 137 | cur.execute(min_album_query) 138 | rows = cur.fetchone() 139 | min = rows['min'] 140 | 141 | cur.execute(max_album_query) 142 | rows = cur.fetchone() 143 | max = rows['max'] 144 | 145 | if min is None: 146 | min = -1 147 | 148 | if max is None: 149 | max = -1 150 | 151 | logger.debug("min max album id: %s to %s", min, max) 152 | 153 | res = min, max 154 | except Exception as e: 155 | res = -1, -1 156 | logger.error("getAlbumMinMaxIds default id defined") 157 | logger.exception(e) 158 | finally: 159 | return res 160 | 161 | def updateAlbumDate(self, albumid, newdate): 162 | """ 163 | Update album date to an arbitrary date 164 | newdate is an epoch timestamp 165 | """ 166 | 167 | res = True 168 | try: 169 | qry = "update lychee_albums set sysstamp= '" + str(newdate) + "' where id=" + str(albumid) 170 | cur = self.db.cursor() 171 | cur.execute(qry) 172 | self.db.commit() 173 | except Exception as e: 174 | logger.exception(e) 175 | res = False 176 | logger.error("updateAlbumDate", Exception) 177 | raise 178 | finally: 179 | return res 180 | 181 | def changeAlbumId(self, oldid, newid): 182 | """ 183 | Change albums id based on album titles (to affect display order) 184 | """ 185 | res = True 186 | photo_query = "update lychee_photos set album = " + str(newid) + " where album = " + str(oldid) 187 | album_query = "update lychee_albums set id = " + str(newid) + " where id = " + str(oldid) 188 | try: 189 | cur = self.db.cursor() 190 | cur.execute(photo_query) 191 | cur.execute(album_query) 192 | self.db.commit() 193 | logger.debug("album id changed: " + str(oldid) + " to " + str(newid)) 194 | except Exception as e: 195 | logger.exception(e) 196 | logger.error("album id changed: " + str(oldid) + " to " + str(newid)) 197 | res = False 198 | finally: 199 | return res 200 | 201 | def loadAlbumList(self): 202 | """ 203 | retrieve all albums in a dictionnary key=title value=id 204 | and put them in self.albumslist 205 | returns self.albumlist 206 | """ 207 | # Load album list 208 | cur = self.db.cursor() 209 | cur.execute("SELECT title,id from lychee_albums") 210 | rows = cur.fetchall() 211 | for row in rows: 212 | self.albumslist[row['title']] = row['id'] 213 | 214 | logger.debug("album list in db:" + str(self.albumslist)) 215 | return self.albumslist 216 | 217 | def albumIdExists(self, album_id): 218 | res = False 219 | try: 220 | cur = self.db.cursor() 221 | cur.execute("select * from lychee_albums where id=%s", (album_id)) 222 | row = cur.fetchall() 223 | if len(row) != 0: 224 | res = True 225 | except Exception as e: 226 | logger.exception(e) 227 | finally: 228 | return res 229 | 230 | def albumExists(self, album): 231 | """ 232 | Check if an album exists based on its name 233 | Parameters: an album properties list. At least the name should be specified 234 | Returns None or the albumid if it exists 235 | """ 236 | logger.debug("exists ? " + str(album)) 237 | if album['name'] in self.albumslist.keys(): 238 | return self.albumslist[album['name']] 239 | else: 240 | return None 241 | 242 | def getAlbumNameFromIdsList(self, list_id): 243 | album_names = '' 244 | try: 245 | albumids = ','.join(list_id) 246 | query = ("select title from lychee_albums where id in(" + albumids + ")") 247 | cur = self.db.cursor() 248 | cur.execute(query) 249 | rows = cur.fetchall() 250 | album_names = [column['title'] for column in rows] 251 | except Exception as e: 252 | album_names = '' 253 | logger.error('impossible to execute ' + query) 254 | logger.exception(e) 255 | finally: 256 | return album_names 257 | 258 | def photoIdExists(self, photoid): 259 | res = None 260 | try: 261 | cur = self.db.cursor() 262 | cur.execute("select id from lychee_photos where id=%s", (photoid)) 263 | row = cur.fetchall() 264 | if len(row) != 0: 265 | logger.debug("photoExistsById %s", row) 266 | res = row[0]['id'] 267 | except Exception as e: 268 | logger.exception(e) 269 | finally: 270 | return res 271 | 272 | def photoExistsByName(self, photo_name): 273 | res = None 274 | try: 275 | cur = self.db.cursor() 276 | cur.execute("select id from lychee_photos where title=%s", (photo_name)) 277 | row = cur.fetchall() 278 | if len(row) != 0: 279 | logger.debug("photoExistsByName %s", row) 280 | res = row[0]['id'] 281 | except Exception as e: 282 | logger.exception(e) 283 | finally: 284 | return res 285 | 286 | def photoExists(self, photo): 287 | """ 288 | Check if a photo already exists in its album based on its original name or checksum 289 | Parameter: 290 | - photo: a valid LycheePhoto object 291 | Returns a boolean 292 | """ 293 | res = False 294 | try: 295 | cur = self.db.cursor() 296 | cur.execute( 297 | "select * from lychee_photos where album=%s AND (title=%s OR checksum=%s)", 298 | (photo.albumid, 299 | photo.originalname, 300 | photo.checksum)) 301 | row = cur.fetchall() 302 | if len(row) != 0: 303 | res = True 304 | 305 | # Add Warning if photo exists in another album 306 | 307 | cur = self.db.cursor() 308 | cur.execute( 309 | "select album from lychee_photos where (title=%s OR checksum=%s)", 310 | (photo.originalname, 311 | photo.checksum)) 312 | rows = cur.fetchall() 313 | album_ids = [r['album'] for r in rows] 314 | if len(album_ids) > 0: 315 | logger.warn( 316 | "a photo with this name: %s or checksum: %s already exists in at least another album: %s", 317 | photo.originalname, 318 | photo.checksum, 319 | self.getAlbumNameFromIdsList(album_ids)) 320 | 321 | except Exception as e: 322 | logger.exception(e) 323 | logger.error("photoExists:", photo.srcfullpath, "won't be added to lychee") 324 | res = True 325 | finally: 326 | return res 327 | 328 | def createAlbum(self, album): 329 | """ 330 | Creates an album 331 | Parameter: 332 | - album: the album properties list, at least the name should be specified 333 | Returns the created albumid or None 334 | """ 335 | album['id'] = str(self.getUniqAlbumId()) 336 | 337 | query = ("insert into lychee_albums (id, title, sysstamp, public, password) values ({},'{}',{},'{}',NULL)".format( 338 | album['id'], 339 | album['name'], 340 | datetime.datetime.now().strftime('%s'), 341 | str(self.conf["publicAlbum"])) 342 | ) 343 | 344 | cur = None 345 | try: 346 | cur = self.db.cursor() 347 | logger.debug("try to createAlbum: %s", query) 348 | # duplicate of previous query to use driver quote protection features 349 | cur.execute("insert into lychee_albums (id, title, sysstamp, public, password) values (%s,%s,%s,%s,NULL)", (album['id'], album['name'], datetime.datetime.now().strftime('%s'), str(self.conf["publicAlbum"]))) 350 | self.db.commit() 351 | 352 | cur.execute("select id from lychee_albums where title=%s", (album['name'])) 353 | row = cur.fetchone() 354 | self.albumslist['name'] = row['id'] 355 | album['id'] = row['id'] 356 | 357 | except Exception as e: 358 | logger.exception(e) 359 | logger.error("createAlbum: %s -> %s", album['name'], str(album)) 360 | album['id'] = None 361 | finally: 362 | return album['id'] 363 | 364 | def eraseAlbum(self, album_id): 365 | """ 366 | Deletes all photos of an album but don't delete the album itself 367 | Parameters: 368 | - album: the album properties list to erase. At least its id must be provided 369 | Return list of the erased photo url 370 | """ 371 | res = [] 372 | query = "delete from lychee_photos where album = " + str(album_id) + '' 373 | selquery = "select url from lychee_photos where album = " + str(album_id) + '' 374 | try: 375 | cur = self.db.cursor() 376 | cur.execute(selquery) 377 | rows = cur.fetchall() 378 | for row in rows: 379 | res.append(row['url']) 380 | cur.execute(query) 381 | self.db.commit() 382 | logger.debug("album photos erased: ", album_id) 383 | except Exception as e: 384 | logger.exception(e) 385 | logger.error("eraseAlbum") 386 | finally: 387 | return res 388 | 389 | def dropAlbum(self, album_id): 390 | res = False 391 | query = "delete from lychee_albums where id = " + str(album_id) + '' 392 | try: 393 | cur = self.db.cursor() 394 | cur.execute(query) 395 | self.db.commit() 396 | logger.debug("album dropped: %s", album_id) 397 | res = True 398 | except Exception as e: 399 | logger.exception(e) 400 | finally: 401 | return res 402 | 403 | def dropPhoto(self, photo_id): 404 | """ delete a photo. parameter: photo_id """ 405 | res = False 406 | query = "delete from lychee_photos where id = " + str(photo_id) + '' 407 | try: 408 | cur = self.db.cursor() 409 | cur.execute(query) 410 | self.db.commit() 411 | logger.debug("photo dropped: %s", photo_id) 412 | res = True 413 | except Exception as e: 414 | logger.exception(e) 415 | finally: 416 | return res 417 | 418 | def get_all_photos(self, album_id=None): 419 | """ 420 | Lists all photos in leeche db (used to delete all files) 421 | Return a photo url list 422 | """ 423 | res = [] 424 | if not(album_id): 425 | selquery = "select id, url, album from lychee_photos" 426 | else: 427 | selquery = "select id, url, album from lychee_photos where album={}".format(album_id) 428 | 429 | try: 430 | cur = self.db.cursor() 431 | cur.execute(selquery) 432 | rows = cur.fetchall() 433 | for row in rows: 434 | p = {} 435 | p['url'] = row['url'] 436 | p['id'] = row['id'] 437 | p['album'] = row['album'] 438 | res.append(p) 439 | except Exception as e: 440 | logger.exception(e) 441 | finally: 442 | return res 443 | 444 | def get_empty_albums(self): 445 | res = [] 446 | try: 447 | # check if exists in db 448 | sql = "select id from lychee_albums where id not in(select distinct album from lychee_photos)" 449 | with self.db.cursor() as cursor: 450 | cursor.execute(sql) 451 | rows = cursor.fetchall() 452 | if rows: 453 | res = [r['id'] for r in rows] 454 | except Exception as e: 455 | logger.exception(e) 456 | res = None 457 | raise e 458 | finally: 459 | return res 460 | 461 | def get_album_ids_titles(self): 462 | res = None 463 | try: 464 | # check if exists in db 465 | sql = "select id, title from lychee_albums" 466 | with self.db.cursor() as cursor: 467 | cursor.execute(sql) 468 | rows = cursor.fetchall() 469 | res = rows 470 | except Exception as e: 471 | # logger.exception(e) 472 | res = None 473 | raise e 474 | finally: 475 | return res 476 | 477 | def addFileToAlbum(self, photo): 478 | """ 479 | Add a photo to an album 480 | Parameter: 481 | - photo: a valid LycheePhoto object 482 | Returns a boolean 483 | """ 484 | res = True 485 | try: 486 | stamp = parse(photo.exif.takedate + ' ' + photo.exif.taketime).strftime('%s') 487 | except Exception as e: 488 | stamp = datetime.datetime.now().strftime('%s') 489 | 490 | query = ("insert into lychee_photos " + 491 | "(id, url, public, type, width, height, " + 492 | "size, star, " + 493 | "thumbUrl, album,iso, aperture, make, " + 494 | "model, shutter, focal, takestamp, " + 495 | "description, title, checksum) " + 496 | "values " + 497 | "({}, '{}', {}, '{}', {}, {}, " + 498 | "'{}', {}, " + 499 | "'{}', '{}', '{}', '{}'," + 500 | " '{}', " + 501 | "'{}', '{}', '{}', '{}', " + 502 | "'{}', %s, '{}')" 503 | ).format(photo.id, photo.url, self.conf["publicAlbum"], photo.type, photo.width, photo.height, 504 | photo.size, photo.star, 505 | photo.thumbUrl, photo.albumid, 506 | photo.exif.iso, 507 | photo.exif.aperture, 508 | photo.exif.make, 509 | photo.exif.model, photo.exif.shutter, photo.exif.focal, stamp, 510 | photo.description, photo.checksum) 511 | try: 512 | logger.debug(query) 513 | cur = self.db.cursor() 514 | res = cur.execute(query, (photo.originalname)) 515 | self.db.commit() 516 | except Exception as e: 517 | logger.exception(e) 518 | logger.error("addFileToAlbum : %s", photo) 519 | logger.error("addFileToAlbum while executing: %s", cur._last_executed) 520 | res = False 521 | finally: 522 | return res 523 | 524 | def reinitAlbumAutoIncrement(self): 525 | 526 | min, max = self.getAlbumMinMaxIds() 527 | if max: 528 | qry = "alter table lychee_albums AUTO_INCREMENT=" + str(max + 1) 529 | try: 530 | cur = self.db.cursor() 531 | cur.execute(qry) 532 | self.db.commit() 533 | logger.debug("reinit auto increment to %s", str(max + 1)) 534 | except Exception as e: 535 | logger.exception(e) 536 | 537 | def close(self): 538 | """ 539 | Close DB Connection 540 | Returns nothing 541 | """ 542 | if self.db: 543 | self.db.close() 544 | 545 | def dropAll(self): 546 | """ 547 | Drop all albums and photos from DB 548 | Returns nothing 549 | """ 550 | try: 551 | cur = self.db.cursor() 552 | cur.execute("delete from lychee_albums") 553 | cur.execute("delete from lychee_photos") 554 | self.db.commit() 555 | except Exception as e: 556 | logger.exception(e) 557 | -------------------------------------------------------------------------------- /lycheesync/lycheemodel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import unicode_literals 4 | from __future__ import print_function 5 | import time 6 | import hashlib 7 | import random 8 | import math 9 | import decimal 10 | from fractions import Fraction 11 | import os 12 | import mimetypes 13 | from PIL import Image 14 | from PIL.ExifTags import TAGS 15 | import datetime 16 | import logging 17 | from dateutil.parser import parse 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | class ExifData: 23 | 24 | """ 25 | Use to store ExifData 26 | """ 27 | 28 | @property 29 | def takedate(self): 30 | return self._takedate 31 | 32 | @takedate.setter 33 | def takedate(self, value): 34 | self._takedate = value.replace(':', '-') 35 | 36 | iso = "" 37 | make = "" 38 | model = "" 39 | shutter = None 40 | aperture = None 41 | exposure = None 42 | focal = None 43 | _takedate = None 44 | taketime = None 45 | orientation = 1 46 | 47 | def __str__(self): 48 | res = "" 49 | res += "iso: " + str(self.iso) + "\n" 50 | res += "aperture: " + str(self.aperture) + "\n" 51 | res += "make: " + str(self.make) + "\n" 52 | res += "model: " + str(self.model) + "\n" 53 | res += "shutter: " + str(self.shutter) + "\n" 54 | res += "exposure: " + str(self.exposure) + "\n" 55 | res += "focal: " + str(self.focal) + "\n" 56 | res += "takedate: " + str(self.takedate) + "\n" 57 | res += "taketime: " + str(self.taketime) + "\n" 58 | res += "orientation: " + str(self.orientation) + "\n" 59 | return res 60 | 61 | 62 | class LycheePhoto: 63 | 64 | """ 65 | Use to store photo data 66 | """ 67 | 68 | originalname = "" # import_name 69 | originalpath = "" 70 | id = "" 71 | albumname = "" 72 | albumid = "" 73 | thumbnailfullpath = "" 74 | thumbnailx2fullpath = "" 75 | title = "" 76 | description = "" 77 | url = "" 78 | public = 0 # private by default 79 | type = "" 80 | width = 0 81 | height = 0 82 | size = "" 83 | star = 0 # no star by default 84 | thumbUrl = "" 85 | srcfullpath = "" 86 | destfullpath = "" 87 | exif = None 88 | _str_datetime = None 89 | checksum = "" 90 | 91 | def convert_strdate_to_timestamp(self, value): 92 | # check parameter type 93 | # logger.debug("convert_strdate input: " + str(value)) 94 | # logger.debug("convert_strdate input_type: " + str(type(value))) 95 | 96 | timestamp = None 97 | # now in epoch time 98 | epoch_now = int(time.time()) 99 | timestamp = epoch_now 100 | 101 | if isinstance(value, int): 102 | timestamp = value 103 | elif isinstance(value, datetime.date): 104 | timestamp = (value - datetime.datetime(1970, 1, 1)).total_seconds() 105 | elif value: 106 | 107 | value = str(value) 108 | 109 | try: 110 | the_date = parse(value) 111 | # works for python 3 112 | # timestamp = the_date.timestamp() 113 | timestamp = time.mktime(the_date.timetuple()) 114 | 115 | except Exception: 116 | logger.warn('model date impossible to parse: ' + str(value)) 117 | timestamp = epoch_now 118 | else: 119 | # Value is None 120 | timestamp = epoch_now 121 | 122 | return timestamp 123 | 124 | @property 125 | def epoch_sysdate(self): 126 | return self.convert_strdate_to_timestamp(self._str_datetime) 127 | 128 | # Compute checksum 129 | def __generateHash(self): 130 | sha1 = hashlib.sha1() 131 | with open(self.srcfullpath, 'rb') as f: 132 | sha1.update(f.read()) 133 | self.checksum = sha1.hexdigest() 134 | 135 | def __init__(self, id, conf, photoname, album): 136 | # Parameters storage 137 | self.conf = conf 138 | self.id = id 139 | self.originalname = photoname 140 | self.originalpath = album['path'] 141 | self.albumid = album['id'] 142 | self.albumname = album['name'] 143 | 144 | # if star in file name, photo is starred 145 | if ('star' in self.originalname) or ('cover' in self.originalname): 146 | self.star = 1 147 | 148 | assert len(self.id) == 14, "id {} is not 14 character long: {}".format(self.id, str(len(self.id))) 149 | 150 | # Compute file storage url 151 | m = hashlib.md5() 152 | m.update(self.id.encode('utf-8')) 153 | crypted = m.hexdigest() 154 | 155 | ext = os.path.splitext(photoname)[1] 156 | self.url = ''.join([crypted, ext]).lower() 157 | self.thumbUrl = self.url 158 | 159 | # src and dest fullpath 160 | self.srcfullpath = os.path.join(self.originalpath, self.originalname) 161 | self.destfullpath = os.path.join(self.conf["lycheepath"], "uploads", "big", self.url) 162 | 163 | # Generate file checksum 164 | self.__generateHash() 165 | 166 | # thumbnails already in place (see makeThumbnail) 167 | 168 | # Auto file some properties 169 | self.type = mimetypes.guess_type(self.originalname, False)[0] 170 | self.size = os.path.getsize(self.srcfullpath) 171 | self.size = str(self.size / 1024) + " KB" 172 | # Default date 173 | takedate = datetime.date.today().isoformat() 174 | taketime = datetime.datetime.now().strftime('%H:%M:%S') 175 | self._str_datetime = takedate + " " + taketime 176 | 177 | # Exif Data Parsing 178 | self.exif = ExifData() 179 | try: 180 | 181 | img = Image.open(self.srcfullpath) 182 | w, h = img.size 183 | self.width = float(w) 184 | self.height = float(h) 185 | 186 | if hasattr(img, '_getexif'): 187 | try: 188 | exifinfo = img._getexif() 189 | except Exception as e: 190 | exifinfo = None 191 | logger.warn('Could not obtain exif info for image: %s', e) 192 | # exifinfo = img.info['exif'] 193 | # logger.debug(exifinfo) 194 | if exifinfo is not None: 195 | for tag, value in exifinfo.items(): 196 | decode = TAGS.get(tag, tag) 197 | if decode == "Orientation": 198 | self.exif.orientation = value 199 | if decode == "Make": 200 | self.exif.make = value 201 | if decode == "MaxApertureValue": 202 | aperture = math.sqrt(2) ** value[0] 203 | try: 204 | aperture = decimal.Decimal(aperture).quantize( 205 | decimal.Decimal('.1'), 206 | rounding=decimal.ROUND_05UP) 207 | except Exception as e: 208 | logger.debug("aperture only a few digit after comma: {}".format(aperture)) 209 | logger.debug(e) 210 | self.exif.aperture = aperture 211 | if decode == "FocalLength": 212 | self.exif.focal = value[0] 213 | if decode == "ISOSpeedRatings": 214 | self.exif.iso = value[0] 215 | if decode == "Model": 216 | self.exif.model = value 217 | if decode == "ExposureTime": 218 | self.exif.exposure = value[0] 219 | if decode == "ShutterSpeedValue": 220 | s = value[0] 221 | s = 2 ** s 222 | s = decimal.Decimal(s).quantize(decimal.Decimal('1'), rounding=decimal.ROUND_05UP) 223 | if s <= 1: 224 | s = decimal.Decimal( 225 | 1 / 226 | float(s)).quantize( 227 | decimal.Decimal('0.1'), 228 | rounding=decimal.ROUND_05UP) 229 | else: 230 | s = "1/" + str(s) 231 | self.exif.shutter = str(s) + " s" 232 | 233 | if decode == "DateTimeOriginal": 234 | try: 235 | self.exif.takedate = value[0].split(" ")[0] 236 | except Exception as e: 237 | logger.warn('invalid takedate: ' + str(value) + ' for ' + self.srcfullpath) 238 | 239 | if decode == "DateTimeOriginal": 240 | try: 241 | self.exif.taketime = value[0].split(" ")[1] 242 | except Exception as e: 243 | logger.warn('invalid taketime: ' + str(value) + ' for ' + self.srcfullpath) 244 | 245 | if decode == "DateTime" and self.exif.takedate is None: 246 | try: 247 | self.exif.takedate = value.split(" ")[0] 248 | except Exception as e: 249 | logger.warn('DT invalid takedate: ' + str(value) + ' for ' + self.srcfullpath) 250 | 251 | if decode == "DateTime" and self.exif.taketime is None: 252 | try: 253 | self.exif.taketime = value.split(" ")[1] 254 | except Exception as e: 255 | logger.warn('DT invalid taketime: ' + str(value) + ' for ' + self.srcfullpath) 256 | 257 | # compute shutter speed 258 | 259 | if not(self.exif.shutter) and self.exif.exposure: 260 | if self.exif.exposure < 1: 261 | e = str(Fraction(self.exif.exposure).limit_denominator()) 262 | else: 263 | e = decimal.Decimal( 264 | self.exif.exposure).quantize( 265 | decimal.Decimal('0.01'), 266 | rounding=decimal.ROUND_05UP) 267 | self.exif.shutter = e 268 | 269 | if self.exif.shutter: 270 | self.exif.shutter = str(self.exif.shutter) + " s" 271 | else: 272 | self.exif.shutter = "" 273 | 274 | if self.exif.exposure: 275 | self.exif.exposure = str(self.exif.exposure) + " s" 276 | else: 277 | self.exif.exposure = "" 278 | 279 | if self.exif.focal: 280 | self.exif.focal = str(self.exif.focal) + " mm" 281 | else: 282 | self.exif.focal = "" 283 | 284 | if self.exif.aperture: 285 | self.exif.aperture = 'F' + str(self.exif.aperture) 286 | else: 287 | self.exif.aperture = "" 288 | 289 | # compute takedate / taketime 290 | if self.exif.takedate: 291 | takedate = self.exif.takedate.replace(':', '-') 292 | taketime = '00:00:00' 293 | 294 | if self.exif.taketime: 295 | taketime = self.exif.taketime 296 | 297 | # add mesurement units 298 | 299 | self._str_datetime = takedate + " " + taketime 300 | 301 | self.description = self._str_datetime 302 | 303 | except IOError as e: 304 | logger.debug('ioerror (corrupted ?): ' + self.srcfullpath) 305 | raise e 306 | 307 | def __str__(self): 308 | res = "" 309 | res += "originalname:" + str(self.originalname) + "\n" 310 | res += "originalpath:" + str(self.originalpath) + "\n" 311 | res += "id:" + str(self.id) + "\n" 312 | res += "albumname:" + str(self.albumname) + "\n" 313 | res += "albumid:" + str(self.albumid) + "\n" 314 | res += "thumbnailfullpath:" + str(self.thumbnailfullpath) + "\n" 315 | res += "thumbnailx2fullpath:" + str(self.thumbnailx2fullpath) + "\n" 316 | res += "title:" + str(self.title) + "\n" 317 | res += "description:" + str(self.description) + "\n" 318 | res += "url:" + str(self.url) + "\n" 319 | res += "public:" + str(self.public) + "\n" 320 | res += "type:" + str(self.type) + "\n" 321 | res += "width:" + str(self.width) + "\n" 322 | res += "height:" + str(self.height) + "\n" 323 | res += "size:" + str(self.size) + "\n" 324 | res += "star:" + str(self.star) + "\n" 325 | res += "thumbUrl:" + str(self.thumbUrl) + "\n" 326 | res += "srcfullpath:" + str(self.srcfullpath) + "\n" 327 | res += "destfullpath:" + str(self.destfullpath) + "\n" 328 | res += "_str_datetime:" + self._str_datetime + "\n" 329 | res += "epoch_sysdate:" + str(self.epoch_sysdate) + "\n" 330 | res += "checksum:" + self.checksum + "\n" 331 | res += "Exif: \n" + str(self.exif) + "\n" 332 | return res 333 | -------------------------------------------------------------------------------- /lycheesync/lycheesyncer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import unicode_literals 4 | from __future__ import print_function 5 | import os 6 | import shutil 7 | import stat 8 | from lycheesync.lycheedao import LycheeDAO 9 | from lycheesync.lycheemodel import LycheePhoto 10 | from lycheesync.utils.configuration import ConfBorg 11 | from PIL import Image 12 | import datetime 13 | import time 14 | import sys 15 | import logging 16 | import piexif 17 | import fnmatch 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | def remove_file(path): 23 | try: 24 | os.remove(path) 25 | except Exception as e: 26 | logger.warn("problem removing: " + path) 27 | logger.debug(e) 28 | 29 | 30 | class LycheeSyncer: 31 | 32 | """ 33 | This class contains the logic behind this program 34 | It consist mainly in filesystem operations 35 | It relies on: 36 | - LycheeDAO for dtabases operations 37 | - LycheePhoto to store (and compute) photos propreties 38 | """ 39 | 40 | conf = {} 41 | 42 | def __init__(self): 43 | """ 44 | Takes a dictionnary of conf as input 45 | """ 46 | borg = ConfBorg() 47 | self.conf = borg.conf 48 | 49 | def getAlbumNameFromPath(self, album): 50 | """ 51 | build a lychee compatible albumname from an albumpath (relative to the srcdir main argument) 52 | Takes an album properties list as input. At least the path sould be specified (relative albumpath) 53 | Returns a string, the lychee album name 54 | """ 55 | # make a list with directory and sub dirs 56 | alb_path_utf8 = album['relpath'] # .decode('UTF-8') 57 | 58 | path = alb_path_utf8.split(os.sep) 59 | 60 | # join the rest: no subfolders in lychee yet 61 | if len(path) > 1: 62 | album['name'] = "_".join(path) 63 | else: 64 | album['name'] = alb_path_utf8 65 | return album['name'] 66 | 67 | def isAPhoto(self, file): 68 | """ 69 | Determine if the filename passed is a photo or not based on the file extension 70 | Takes a string as input (a file name) 71 | Returns a boolean 72 | """ 73 | validimgext = ['.jpg', '.jpeg', '.gif', '.png'] 74 | ext = os.path.splitext(file)[-1].lower() 75 | return (ext in validimgext) 76 | 77 | def albumExists(self, album): 78 | """ 79 | Takes an album properties list as input. At least the relpath sould be specified (relative albumpath) 80 | Returns an albumid or None if album does not exists 81 | """ 82 | 83 | def createAlbum(self, album): 84 | """ 85 | Creates an album 86 | Inputs: 87 | - album: an album properties list. at least path should be specified (relative albumpath) 88 | Returns an albumid or None if album does not exists 89 | """ 90 | album['id'] = None 91 | if album['name'] != "": 92 | album['id'] = self.dao.createAlbum(album) 93 | return album['id'] 94 | 95 | def thumbIt(self, res, photo, destinationpath, destfile): 96 | """ 97 | Create the thumbnail of a given photo 98 | Parameters: 99 | - res: should be a set of h and v res (640, 480) 100 | - photo: a valid LycheePhoto object 101 | - destinationpath: a string the destination full path of the thumbnail (without filename) 102 | - destfile: the thumbnail filename 103 | Returns the fullpath of the thuumbnail 104 | """ 105 | 106 | if photo.width > photo.height: 107 | delta = photo.width - photo.height 108 | left = int(delta / 2) 109 | upper = 0 110 | right = int(photo.height + left) 111 | lower = int(photo.height) 112 | else: 113 | delta = photo.height - photo.width 114 | left = 0 115 | upper = int(delta / 2) 116 | right = int(photo.width) 117 | lower = int(photo.width + upper) 118 | 119 | destimage = os.path.join(destinationpath, destfile) 120 | try: 121 | img = Image.open(photo.destfullpath) 122 | except Exception as e: 123 | logger.exception(e) 124 | logger.error("ioerror (corrupted file?): " + photo.srcfullpath) 125 | raise 126 | 127 | img = img.crop((left, upper, right, lower)) 128 | img.thumbnail(res, Image.ANTIALIAS) 129 | img.save(destimage, quality=99) 130 | return destimage 131 | 132 | def makeThumbnail(self, photo): 133 | """ 134 | Make the 2 thumbnails needed by Lychee for a given photo 135 | and store their path in the LycheePhoto object 136 | Parameters: 137 | - photo: a valid LycheePhoto object 138 | returns nothing 139 | """ 140 | # set thumbnail size 141 | sizes = [(200, 200), (400, 400)] 142 | # insert @2x in big thumbnail file name 143 | filesplit = os.path.splitext(photo.url) 144 | destfiles = [photo.url, ''.join([filesplit[0], "@2x", filesplit[1]]).lower()] 145 | # compute destination path 146 | destpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb") 147 | # make thumbnails 148 | photo.thumbnailfullpath = self.thumbIt(sizes[0], photo, destpath, destfiles[0]) 149 | photo.thumbnailx2fullpath = self.thumbIt(sizes[1], photo, destpath, destfiles[1]) 150 | 151 | def copyFileToLychee(self, photo): 152 | """ 153 | add a file to an album, the albumid must be previously stored in the LycheePhoto parameter 154 | Parameters: 155 | - photo: a valid LycheePhoto object 156 | Returns True if everything went ok 157 | """ 158 | res = False 159 | 160 | try: 161 | # copy photo 162 | if self.conf['link']: 163 | os.symlink(photo.srcfullpath, photo.destfullpath) 164 | else: 165 | shutil.copy(photo.srcfullpath, photo.destfullpath) 166 | # adjust right (chmod/chown) 167 | try: 168 | os.lchown(photo.destfullpath, -1, self.conf['gid']) 169 | 170 | if not(self.conf['link']): 171 | st = os.stat(photo.destfullpath) 172 | os.chmod(photo.destfullpath, st.st_mode | stat.S_IRWXU | stat.S_IRWXG) 173 | else: 174 | st = os.stat(photo.srcfullpath) 175 | os.chmod(photo.srcfullpath, st.st_mode | stat.S_IROTH) 176 | 177 | except Exception as e: 178 | if self.conf["verbose"]: 179 | logger.warn( 180 | "chgrp error, check file permission for %s fix: eventually adjust source file permissions", 181 | photo.destfullpath) 182 | res = True 183 | 184 | except Exception as e: 185 | logger.exception(e) 186 | res = False 187 | 188 | return res 189 | 190 | def deleteFiles(self, filelist): 191 | """ 192 | Delete files in the Lychee file tree (uploads/big and uploads/thumbnails) 193 | Give it the file name and it will delete relatives files and thumbnails 194 | Parameters: 195 | - filelist: a list of filenames 196 | Returns nothing 197 | """ 198 | 199 | for url in filelist: 200 | if self.isAPhoto(url): 201 | thumbpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb", url) 202 | filesplit = os.path.splitext(url) 203 | thumb2path = ''.join([filesplit[0], "@2x", filesplit[1]]).lower() 204 | thumb2path = os.path.join(self.conf["lycheepath"], "uploads", "thumb", thumb2path) 205 | bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", url) 206 | remove_file(thumbpath) 207 | remove_file(thumb2path) 208 | remove_file(bigpath) 209 | 210 | def adjustRotation(self, photo): 211 | """ 212 | Rotates photos according to the exif orientaion tag 213 | Returns nothing DOIT BEFORE THUMBNAILS !!! 214 | """ 215 | 216 | if photo.exif.orientation != 1: 217 | 218 | img = Image.open(photo.destfullpath) 219 | if "exif" in img.info: 220 | exif_dict = piexif.load(img.info["exif"]) 221 | 222 | if piexif.ImageIFD.Orientation in exif_dict["0th"]: 223 | orientation = exif_dict["0th"][piexif.ImageIFD.Orientation] 224 | 225 | if orientation == 2: 226 | img = img.transpose(Image.FLIP_LEFT_RIGHT) 227 | elif orientation == 3: 228 | img = img.rotate(180) 229 | elif orientation == 4: 230 | img = img.rotate(180).transpose(Image.FLIP_LEFT_RIGHT) 231 | elif orientation == 5: 232 | img = img.rotate(-90, expand=True).transpose(Image.FLIP_LEFT_RIGHT) 233 | elif orientation == 6: 234 | img = img.rotate(-90, expand=True) 235 | elif orientation == 7: 236 | img = img.rotate(90, expand=True).transpose(Image.FLIP_LEFT_RIGHT) 237 | elif orientation == 8: 238 | img = img.rotate(90, expand=True) 239 | else: 240 | if orientation != 1: 241 | logger.warn("Orientation not defined {} for photo {}".format(orientation, photo.title)) 242 | 243 | if orientation in [5, 6, 7, 8]: 244 | # invert width and height 245 | h = photo.height 246 | w = photo.width 247 | photo.height = w 248 | photo.width = h 249 | exif_dict["0th"][piexif.ImageIFD.Orientation] = 1 250 | exif_bytes = piexif.dump(exif_dict) 251 | img.save(photo.destfullpath, exif=exif_bytes, quality=99) 252 | img.close() 253 | 254 | def reorderalbumids(self, albums): 255 | 256 | # sort albums by title 257 | def getName(album): 258 | return album['name'] 259 | 260 | sortedalbums = sorted(albums, key=getName) 261 | 262 | # count albums 263 | nbalbum = len(albums) 264 | # get higher album id + 1 as a first new album id 265 | min, max = self.dao.getAlbumMinMaxIds() 266 | 267 | if min and max: 268 | 269 | if nbalbum + 1 < min: 270 | newid = 1 271 | else: 272 | newid = max + 1 273 | 274 | for a in sortedalbums: 275 | self.dao.changeAlbumId(a['id'], newid) 276 | newid = newid + 1 277 | 278 | def updateAlbumsDate(self, albums): 279 | now = datetime.datetime.now() 280 | last2min = now - datetime.timedelta(minutes=2) 281 | last2min_epoch = int((last2min - datetime.datetime(1970, 1, 1)).total_seconds()) 282 | 283 | for a in albums: 284 | try: 285 | # get photos with a real date (not just now) 286 | datelist = None 287 | 288 | if len(a['photos']) > 0: 289 | 290 | datelist = [ 291 | photo.epoch_sysdate for photo in a['photos'] if photo.epoch_sysdate < last2min_epoch] 292 | 293 | if datelist is not None and len(datelist) > 0: 294 | newdate = max(datelist) 295 | self.dao.updateAlbumDate(a['id'], newdate) 296 | logger.debug( 297 | "album %s sysstamp changed to: %s ", a['name'], str( 298 | time.strftime( 299 | '%Y-%m-%d %H:%M:%S', time.localtime(newdate)))) 300 | except Exception as e: 301 | logger.exception(e) 302 | logger.error("updating album date for album:" + a['name'], e) 303 | 304 | def deleteAllFiles(self): 305 | """ 306 | Deletes every photo file in Lychee 307 | Returns nothing 308 | """ 309 | filelist = [] 310 | photopath = os.path.join(self.conf["lycheepath"], "uploads", "big") 311 | filelist = [f for f in os.listdir(photopath)] 312 | self.deleteFiles(filelist) 313 | 314 | def deletePhotos(self, photo_list): 315 | "photo_list: a list of dictionnary containing key url and id" 316 | if len(photo_list) > 0: 317 | url_list = [p['url'] for p in photo_list] 318 | self.deleteFiles(url_list) 319 | for p in photo_list: 320 | self.dao.dropPhoto(p['id']) 321 | 322 | def sync(self): 323 | """ 324 | Program main loop 325 | Scans files to add in the sourcedirectory and add them to Lychee 326 | according to the conf file and given parameters 327 | Returns nothing 328 | """ 329 | 330 | # Connect db 331 | # and drop it if dropdb activated 332 | self.dao = LycheeDAO(self.conf) 333 | 334 | if self.conf['dropdb']: 335 | self.deleteAllFiles() 336 | 337 | # Load db 338 | 339 | createdalbums = 0 340 | discoveredphotos = 0 341 | importedphotos = 0 342 | album = {} 343 | albums = [] 344 | 345 | album_name_max_width = self.dao.getAlbumNameDBWidth() 346 | 347 | # walkthroug each file / dir of the srcdir 348 | for root, dirs, files in os.walk(self.conf['srcdir']): 349 | 350 | if sys.version_info.major == 2: 351 | try: 352 | root = root.decode('UTF-8') 353 | except Exception as e: 354 | logger.error(e) 355 | # Init album data 356 | album['id'] = None 357 | album['name'] = None 358 | album['path'] = None 359 | album['relpath'] = None # path relative to srcdir 360 | album['photos'] = [] # path relative to srcdir 361 | 362 | # if a there is at least one photo in the files 363 | if any([self.isAPhoto(f) for f in files]): 364 | album['path'] = root 365 | 366 | # Skip any albums that matches one of the exluded patterns 367 | if any([True for pattern in self.conf['excludeAlbums'] if fnmatch.fnmatch(root, pattern)]): 368 | logger.info("Skipping excluded album {}".format(root)) 369 | continue 370 | 371 | # don't know what to do with theses photo 372 | # and don't wan't to create a default album 373 | if album['path'] == self.conf['srcdir']: 374 | msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format( 375 | root) 376 | logger.warn(msg) 377 | continue 378 | 379 | # Fill in other album properties 380 | # albumnames start at srcdir (to avoid absolute path albumname) 381 | album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir']) 382 | album['name'] = self.getAlbumNameFromPath(album) 383 | 384 | if len(album['name']) > album_name_max_width: 385 | logger.warn("album name too long, will be truncated " + album['name']) 386 | album['name'] = album['name'][0:album_name_max_width] 387 | logger.warn("album name is now " + album['name']) 388 | 389 | album['id'] = self.dao.albumExists(album) 390 | 391 | if self.conf['replace'] and album['id']: 392 | # drop album photos 393 | filelist = self.dao.eraseAlbum(album['id']) 394 | self.deleteFiles(filelist) 395 | assert self.dao.dropAlbum(album['id']) 396 | # Album should be recreated 397 | album['id'] = False 398 | 399 | if not(album['id']): 400 | # create album 401 | album['id'] = self.createAlbum(album) 402 | 403 | if not(album['id']): 404 | logger.error("didn't manage to create album for: " + album['relpath']) 405 | continue 406 | else: 407 | logger.info("############ Album created: %s", album['name']) 408 | 409 | createdalbums += 1 410 | 411 | # Albums are created or emptied, now take care of photos 412 | for f in sorted(files): 413 | 414 | if self.isAPhoto(f): 415 | try: 416 | discoveredphotos += 1 417 | error = False 418 | logger.debug( 419 | "**** Trying to add to lychee album %s: %s", 420 | album['name'], 421 | os.path.join( 422 | root, 423 | f)) 424 | # corruption detected here by launching exception 425 | pid = self.dao.getUniqPhotoId() 426 | photo = LycheePhoto(pid, self.conf, f, album) 427 | if not(self.dao.photoExists(photo)): 428 | res = self.copyFileToLychee(photo) 429 | self.adjustRotation(photo) 430 | self.makeThumbnail(photo) 431 | res = self.dao.addFileToAlbum(photo) 432 | # increment counter 433 | if res: 434 | importedphotos += 1 435 | album['photos'].append(photo) 436 | else: 437 | error = True 438 | logger.error( 439 | "while adding to album: %s photo: %s", 440 | album['name'], 441 | photo.srcfullpath) 442 | else: 443 | logger.error( 444 | "photo already exists in this album with same name or same checksum: %s it won't be added to lychee", 445 | photo.srcfullpath) 446 | error = True 447 | except Exception as e: 448 | 449 | logger.exception(e) 450 | logger.error("could not add %s to album %s", f, album['name']) 451 | error = True 452 | finally: 453 | if not(error): 454 | logger.info( 455 | "**** Successfully added %s to lychee album %s", 456 | os.path.join( 457 | root, 458 | f), 459 | album['name']) 460 | 461 | a = album.copy() 462 | albums.append(a) 463 | 464 | self.updateAlbumsDate(albums) 465 | if self.conf['sort']: 466 | self.reorderalbumids(albums) 467 | self.dao.reinitAlbumAutoIncrement() 468 | 469 | if self.conf['sanity']: 470 | 471 | logger.info("************ SANITY CHECK *************") 472 | # get All Photos albums 473 | photos = self.dao.get_all_photos() 474 | albums = [p['album'] for p in photos] 475 | albums = set(albums) 476 | 477 | # for each album 478 | for a_id in albums: 479 | # check if it exists, if not remove photos 480 | if not(self.dao.albumIdExists(a_id)): 481 | to_delete = self.dao.get_all_photos(a_id) 482 | self.dao.eraseAlbum(a_id) 483 | file_list = [p['url'] for p in to_delete] 484 | self.deleteFiles(file_list) 485 | 486 | # get All Photos 487 | photos = self.dao.get_all_photos() 488 | 489 | to_delete = [] 490 | # for each photo 491 | for p in photos: 492 | delete_photo = False 493 | # check if big exists 494 | bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", p['url']) 495 | 496 | # if big is a link check if it's an orphan 497 | # file does not exists 498 | if not(os.path.lexists(bigpath)): 499 | logger.error("File does not exists %s: will be delete in db", bigpath) 500 | delete_photo = True 501 | # broken link 502 | elif not(os.path.exists(bigpath)): 503 | logger.error("Link is broken: %s will be delete in db", bigpath) 504 | delete_photo = True 505 | 506 | if not(delete_photo): 507 | # TODO: check if thumbnail exists 508 | pass 509 | else: 510 | # if any of it is False remove and log 511 | to_delete.append(p) 512 | 513 | self.deletePhotos(to_delete) 514 | 515 | # Detect broken symlinks / orphan files 516 | for root, dirs, files in os.walk(os.path.join(self.conf['lycheepath'], 'uploads', 'big')): 517 | 518 | for f in files: 519 | logger.debug("check orphan: %s", f) 520 | file_name = os.path.basename(f) 521 | # check if DB photo exists 522 | if not self.dao.photoExistsByName(file_name): 523 | # if not delete photo (or link) 524 | self.deleteFiles([file_name]) 525 | logger.info("%s deleted. Wasn't existing in DB", f) 526 | 527 | # if broken link 528 | if os.path.lexists(f) and not(os.path.exists(f)): 529 | id = self.dao.photoExistsByName(file_name) 530 | # if exists in db 531 | if id: 532 | ps = {} 533 | ps['id'] = id 534 | ps['url'] = file_name 535 | self.deletePhotos([ps]) 536 | else: 537 | self.deleteFiles([file_name]) 538 | logger.info("%s deleted. Was a broken link", f) 539 | 540 | # drop empty albums 541 | empty = self.dao.get_empty_albums() 542 | if empty: 543 | for e in empty: 544 | self.dao.dropAlbum(e) 545 | 546 | self.dao.close() 547 | 548 | # Final report 549 | logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") 550 | logger.info("Directory scanned:" + self.conf['srcdir']) 551 | logger.info("Created albums: " + str(createdalbums)) 552 | if (importedphotos == discoveredphotos): 553 | logger.info(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered") 554 | else: 555 | logger.error(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered") 556 | logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") 557 | -------------------------------------------------------------------------------- /lycheesync/sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | # from __future__ import unicode_literals 6 | from lycheesync.lycheesyncer import LycheeSyncer 7 | from lycheesync.update_scripts import inf_to_lychee_2_6_2 8 | import logging.config 9 | import click 10 | import os 11 | import sys 12 | import pwd 13 | import grp 14 | 15 | from lycheesync.utils.boilerplatecode import script_init 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | @click.command() 21 | @click.option('-v', '--verbose', is_flag=True, help='Program verbosity.') 22 | @click.option('-n', '--normal', 'exclusive_mode', flag_value='normal', 23 | default=True, help='normal mode exclusive with replace and delete mode') 24 | @click.option('-r', '--replace', 'exclusive_mode', flag_value='replace', 25 | default=False, help='delete mode exclusive with replace mode and normal') 26 | @click.option('-d', '--dropdb', 'exclusive_mode', flag_value='delete', 27 | default=False, help='delete mode exclusive with replace and normal mode') 28 | @click.option('-s', '--sort_album_by_name', is_flag=True, help='Sort album by name') 29 | @click.option('-c', '--sanitycheck', is_flag=True, help='Sort album by name') 30 | @click.option('-l', '--link', is_flag=True, help="Don't copy files create link instead") 31 | @click.option('-u26', '--updatedb26', is_flag=True, 32 | help="Update lycheesync added data in lychee db to the lychee 2.6.2 required values") 33 | @click.argument('imagedirpath', metavar='PHOTO_DIRECTORY_ROOT', 34 | type=click.Path(exists=True, resolve_path=True)) 35 | @click.argument('lycheepath', metavar='PATH_TO_LYCHEE_INSTALL', 36 | type=click.Path(exists=True, resolve_path=True)) 37 | @click.argument('confpath', metavar='PATH_TO_YOUR_CONFIG_FILE', 38 | type=click.Path(exists=True, resolve_path=True)) 39 | # checks file existence and attributes 40 | # @click.argument('file2', type=click.Path(exists=True, file_okay=True, dir_okay=False, writable=False, readable=True, resolve_path=True)) 41 | def main(verbose, exclusive_mode, sort_album_by_name, sanitycheck, link, updatedb26, imagedirpath, lycheepath, confpath): 42 | """Lycheesync 43 | 44 | A script to synchronize any directory containing photos with Lychee. 45 | Source directory should be on the same host than Lychee's 46 | """ 47 | 48 | if sys.version_info.major == 2: 49 | imagedirpath = imagedirpath.decode('UTF-8') 50 | lycheepath = lycheepath.decode('UTF-8') 51 | confpath = confpath.decode('UTF-8') 52 | 53 | conf_data = {} 54 | conf_data['verbose'] = verbose 55 | conf_data["srcdir"] = imagedirpath 56 | conf_data["lycheepath"] = lycheepath 57 | conf_data['confpath'] = confpath 58 | conf_data["dropdb"] = False 59 | conf_data["replace"] = False 60 | 61 | if exclusive_mode == "delete": 62 | conf_data["dropdb"] = True 63 | elif exclusive_mode == "replace": 64 | conf_data["replace"] = True 65 | 66 | conf_data["user"] = None 67 | conf_data["group"] = None 68 | conf_data["uid"] = None 69 | conf_data["gid"] = None 70 | conf_data["sort"] = sort_album_by_name 71 | if sanitycheck: 72 | logger.info("!!!!!!!!!!!!!!!! SANITY ON") 73 | else: 74 | logger.info("!!!!!!!!!!!!!!!! SANITY OFF") 75 | conf_data["sanity"] = sanitycheck 76 | conf_data["link"] = link 77 | # if conf_data["dropdb"]: 78 | # conf_data["sort"] = True 79 | 80 | # read permission of the lycheepath directory to apply it to the uploade photos 81 | img_path = os.path.join(conf_data["lycheepath"], "uploads", "big") 82 | stat_info = os.stat(img_path) 83 | uid = stat_info.st_uid 84 | gid = stat_info.st_gid 85 | 86 | user = pwd.getpwuid(uid)[0] 87 | group = grp.getgrgid(gid)[0] 88 | 89 | conf_data["user"] = user 90 | conf_data["group"] = group 91 | conf_data["uid"] = uid 92 | conf_data["gid"] = gid 93 | 94 | script_init(conf_data) 95 | 96 | # DB update 97 | if updatedb26: 98 | inf_to_lychee_2_6_2.updatedb(conf_data) 99 | 100 | logger.info("=================== start adding to lychee ==================") 101 | try: 102 | 103 | # DELEGATE WORK TO LYCHEESYNCER 104 | s = LycheeSyncer() 105 | s.sync() 106 | 107 | except Exception: 108 | logger.exception('Failed to run batch') 109 | logger.error("=================== script ended with errors ==================") 110 | 111 | else: 112 | logger.info("=================== script successfully ended ==================") 113 | 114 | 115 | if __name__ == '__main__': 116 | main() 117 | -------------------------------------------------------------------------------- /lycheesync/update_scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/lycheesync/update_scripts/__init__.py -------------------------------------------------------------------------------- /lycheesync/update_scripts/inf_to_lychee_2_6_2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | import os 5 | import pwd 6 | import grp 7 | from lycheesync.lycheesyncer import LycheeSyncer 8 | import pymysql 9 | import hashlib 10 | import stat 11 | import traceback 12 | 13 | 14 | # Compute checksum 15 | def __generateHash(filepath): 16 | checksum = None 17 | sha1 = hashlib.sha1() 18 | with open(filepath, 'rb') as f: 19 | sha1.update(f.read()) 20 | checksum = sha1.hexdigest() 21 | return checksum 22 | 23 | 24 | def updatedb(conf_data): 25 | print("updatedb") 26 | 27 | # read permission of the lycheepath directory to apply it to the uploade photos 28 | upload_dir = os.path.join(conf_data["lycheepath"], "uploads") 29 | stat_info = os.stat(upload_dir) 30 | uid = stat_info.st_uid 31 | gid = stat_info.st_gid 32 | 33 | user = pwd.getpwuid(uid)[0] 34 | group = grp.getgrgid(gid)[0] 35 | 36 | conf_data["user"] = user 37 | conf_data["group"] = group 38 | conf_data["uid"] = uid 39 | conf_data["gid"] = gid 40 | 41 | syncer = LycheeSyncer(conf_data) 42 | 43 | # for each file in upload fix the permissions 44 | for root, dirs, files in os.walk(upload_dir): 45 | for f in files: 46 | if syncer.isAPhoto(f): 47 | # adjust permissions 48 | filepath = os.path.join(root, f) 49 | os.chown(filepath, int(uid), int(gid)) 50 | st = os.stat(filepath) 51 | os.chmod(filepath, st.st_mode | stat.S_IRWXU | stat.S_IRWXG) 52 | print("Changed permission for " + str(f)) 53 | 54 | # connect to db 55 | try: 56 | db = pymysql.connect(host=conf_data["dbHost"], 57 | user=conf_data["dbUser"], 58 | passwd=conf_data["dbPassword"], 59 | db=conf_data["db"]) 60 | 61 | # get photo list 62 | cur = db.cursor() 63 | cur.execute("SELECT id, url from lychee_photos") 64 | rows = cur.fetchall() 65 | for row in rows: 66 | 67 | pid = row[0] 68 | url = row[1] 69 | 70 | photo_path = os.path.join(upload_dir, "big", url) 71 | chksum = __generateHash(photo_path) 72 | # for each photo in db recalculate checksum 73 | qry = "update lychee_photos set checksum= '" + chksum + "' where id=" + str(pid) 74 | try: 75 | cur = db.cursor() 76 | cur.execute(qry) 77 | db.commit() 78 | print("INFO photo checksum changed to: ", chksum) 79 | except Exception: 80 | print("checksum modification failed for photo:" + id, Exception) 81 | traceback.print_exc() 82 | 83 | print("******************************") 84 | print("SUCCESS") 85 | print("******************************") 86 | except Exception: 87 | traceback.print_exc() 88 | finally: 89 | db.close() 90 | -------------------------------------------------------------------------------- /lycheesync/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/lycheesync/utils/__init__.py -------------------------------------------------------------------------------- /lycheesync/utils/boilerplatecode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | import json 4 | import os 5 | import logging 6 | from lycheesync.utils.configuration import ConfBorg 7 | import sys 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def init_loggers(logconf, verbose=False): 13 | with open(logconf, 'rt') as f: 14 | config = json.load(f) 15 | logging.config.dictConfig(config) 16 | logger.debug("**** logging conf -> read from: " + logconf) 17 | if verbose: 18 | logging.getLogger().setLevel(logging.DEBUG) 19 | for h in logging.getLogger().handlers: 20 | if h.name == "stream_handler": 21 | h.setLevel(logging.DEBUG) 22 | 23 | 24 | def script_init(cli_args): 25 | """ 26 | - will initialize a ConfBorg object containing cli arguments, configutation file elements 27 | - will initialize loggers 28 | """ 29 | 30 | root_level = ".." 31 | 32 | # compute log file absolute path 33 | pathname = os.path.dirname(sys.argv[0]) 34 | full_path = os.path.abspath(pathname) 35 | 36 | # root level is different if main.py or sync.py is used to launch script 37 | log_conf_path = os.path.join(full_path, root_level, "ressources", 'logging.json') 38 | log_conf_path2 = os.path.join(full_path, "ressources", 'logging.json') 39 | 40 | # append path to configuration 41 | cli_args['full_path'] = full_path 42 | 43 | # read log configuration 44 | if os.path.exists(log_conf_path): 45 | init_loggers(log_conf_path, cli_args['verbose']) 46 | elif os.path.exists(log_conf_path2): 47 | init_loggers(log_conf_path2, cli_args['verbose']) 48 | else: 49 | # default value 50 | logging.basicConfig(level=logging.DEBUG) 51 | logging.warn("**** logging conf -> default conf") 52 | 53 | # read application configuration 54 | if os.path.exists(cli_args['confpath']): 55 | with open(cli_args['confpath'], 'rt') as f: 56 | conf = json.load(f) 57 | else: 58 | logger.warn("**** Loading default conf in ressources/conf.json") 59 | conf_path = os.path.join(full_path, root_level, "ressources", 'conf.json') 60 | if os.path.exists(conf_path): 61 | with open(conf_path, 'rt') as f: 62 | conf = json.load(f) 63 | 64 | # initialize conf with items loaded from conf file AND command lines arguments 65 | # cli args have priority over configuration file 66 | z = {} 67 | z = conf.copy() 68 | z.update(cli_args) 69 | borg = ConfBorg(z) 70 | logger.debug("**** loaded configuration: ") 71 | logger.debug("**** " + borg.pretty) 72 | -------------------------------------------------------------------------------- /lycheesync/utils/configuration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pprint import pformat 5 | 6 | 7 | class Borg: 8 | _shared_state = {} 9 | 10 | def __init__(self): 11 | self.__dict__ = self._shared_state 12 | 13 | 14 | class ConfBorg(Borg): 15 | 16 | isinitialized = False 17 | 18 | def __init__(self, confdic=None, force_init=False): 19 | Borg.__init__(self) 20 | 21 | if force_init: 22 | self.isinitialized = False 23 | return 24 | 25 | if not (self.isinitialized): 26 | if confdic: 27 | self.confdic = confdic 28 | self.isinitialized = True 29 | else: 30 | raise Exception('ConfBorg not initialized') 31 | 32 | def __str__(self): 33 | return str(self.val) 34 | 35 | @property 36 | def pretty(self): 37 | return "\n" + pformat(self.confdic) 38 | 39 | @property 40 | def conf(self): 41 | return self.confdic 42 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from lycheesync.sync import main 5 | 6 | if __name__ == '__main__': 7 | import sys 8 | print(sys.argv[1:]) 9 | main(sys.argv[1:]) 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pymysql==0.7.1 2 | click==6.2 3 | pillow==3.0.0 4 | python-dateutil 5 | pytest==2.7.3 6 | pytest-cov==2.2.0 7 | pytest-pep8==1.0.6 8 | piexif==1.0.3 9 | -------------------------------------------------------------------------------- /ressources/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "db":"lycheetest", 3 | "dbUser":"lycheetest", 4 | "dbPassword":"lycheetest", 5 | "dbHost":"localhost", 6 | "dbSocket":"/var/run/mysqld/mysqld.sock", 7 | "thumbQuality":80, 8 | "publicAlbum": 0, 9 | "excludeAlbums": [ 10 | ] 11 | } 12 | 13 | -------------------------------------------------------------------------------- /ressources/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "disable_existing_loggers": false, 4 | 5 | "formatters": { 6 | "simple": { 7 | "format": "%(levelname)s;%(asctime)s; %(name)s; %(message)s" 8 | } 9 | }, 10 | 11 | "handlers": { 12 | "stream_handler": { 13 | "class": "logging.StreamHandler", 14 | "level": "INFO", 15 | "formatter": "simple", 16 | "stream": "ext://sys.stdout" 17 | 18 | }, 19 | 20 | "basic_file_handler":{ 21 | "class": "logging.FileHandler", 22 | "level": "DEBUG", 23 | "formatter": "simple", 24 | "filename": "logs/lycheesync.log", 25 | "mode" : "w", 26 | "encoding": "utf-8" 27 | 28 | } 29 | }, 30 | 31 | "loggers": { 32 | "lycheesync": { 33 | "level": "DEBUG", 34 | "propagate": "no" 35 | }, 36 | "__main__": { 37 | "level": "DEBUG", 38 | "propagate": "no" 39 | } 40 | }, 41 | 42 | "root": { 43 | "level": "DEBUG", 44 | "handlers": ["stream_handler", "basic_file_handler"] 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /ressources/lychee.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.5.46, for debian-linux-gnu (x86_64) 2 | -- 3 | -- Host: localhost Database: lychee_ci 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.46-0ubuntu0.14.04.2 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `lychee_ci` 20 | -- 21 | 22 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `lychee_ci` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `lychee_ci`; 25 | 26 | -- 27 | -- Table structure for table `lychee_albums` 28 | -- 29 | 30 | DROP TABLE IF EXISTS `lychee_albums`; 31 | /*!40101 SET @saved_cs_client = @@character_set_client */; 32 | /*!40101 SET character_set_client = utf8 */; 33 | CREATE TABLE `lychee_albums` ( 34 | `id` bigint(14) NOT NULL, 35 | `title` varchar(100) NOT NULL DEFAULT '', 36 | `description` varchar(1000) DEFAULT '', 37 | `sysstamp` int(11) NOT NULL, 38 | `public` tinyint(1) NOT NULL DEFAULT '0', 39 | `visible` tinyint(1) NOT NULL DEFAULT '1', 40 | `downloadable` tinyint(1) NOT NULL DEFAULT '0', 41 | `password` varchar(100) DEFAULT NULL, 42 | PRIMARY KEY (`id`) 43 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 44 | /*!40101 SET character_set_client = @saved_cs_client */; 45 | 46 | -- 47 | -- Dumping data for table `lychee_albums` 48 | -- 49 | 50 | LOCK TABLES `lychee_albums` WRITE; 51 | /*!40000 ALTER TABLE `lychee_albums` DISABLE KEYS */; 52 | /*!40000 ALTER TABLE `lychee_albums` ENABLE KEYS */; 53 | UNLOCK TABLES; 54 | 55 | -- 56 | -- Table structure for table `lychee_log` 57 | -- 58 | 59 | DROP TABLE IF EXISTS `lychee_log`; 60 | /*!40101 SET @saved_cs_client = @@character_set_client */; 61 | /*!40101 SET character_set_client = utf8 */; 62 | CREATE TABLE `lychee_log` ( 63 | `id` int(11) NOT NULL AUTO_INCREMENT, 64 | `time` int(11) NOT NULL, 65 | `type` varchar(11) NOT NULL, 66 | `function` varchar(100) NOT NULL, 67 | `line` int(11) NOT NULL, 68 | `text` text, 69 | PRIMARY KEY (`id`) 70 | ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8; 71 | /*!40101 SET character_set_client = @saved_cs_client */; 72 | 73 | -- 74 | -- Dumping data for table `lychee_log` 75 | -- 76 | 77 | LOCK TABLES `lychee_log` WRITE; 78 | /*!40000 ALTER TABLE `lychee_log` DISABLE KEYS */; 79 | /*!40000 ALTER TABLE `lychee_log` ENABLE KEYS */; 80 | UNLOCK TABLES; 81 | 82 | -- 83 | -- Table structure for table `lychee_photos` 84 | -- 85 | 86 | DROP TABLE IF EXISTS `lychee_photos`; 87 | /*!40101 SET @saved_cs_client = @@character_set_client */; 88 | /*!40101 SET character_set_client = utf8 */; 89 | CREATE TABLE `lychee_photos` ( 90 | `id` bigint(14) NOT NULL, 91 | `title` varchar(100) NOT NULL DEFAULT '', 92 | `description` varchar(1000) DEFAULT '', 93 | `url` varchar(100) NOT NULL, 94 | `tags` varchar(1000) NOT NULL DEFAULT '', 95 | `public` tinyint(1) NOT NULL, 96 | `type` varchar(10) NOT NULL, 97 | `width` int(11) NOT NULL, 98 | `height` int(11) NOT NULL, 99 | `size` varchar(20) NOT NULL, 100 | `iso` varchar(15) NOT NULL, 101 | `aperture` varchar(20) NOT NULL, 102 | `make` varchar(50) NOT NULL, 103 | `model` varchar(50) NOT NULL, 104 | `shutter` varchar(30) NOT NULL, 105 | `focal` varchar(20) NOT NULL, 106 | `takestamp` int(11) DEFAULT NULL, 107 | `star` tinyint(1) NOT NULL, 108 | `thumbUrl` varchar(50) NOT NULL, 109 | `album` varchar(30) NOT NULL DEFAULT '0', 110 | `checksum` varchar(100) DEFAULT NULL, 111 | `medium` tinyint(1) NOT NULL DEFAULT '0', 112 | PRIMARY KEY (`id`) 113 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 114 | /*!40101 SET character_set_client = @saved_cs_client */; 115 | 116 | -- 117 | -- Dumping data for table `lychee_photos` 118 | -- 119 | 120 | LOCK TABLES `lychee_photos` WRITE; 121 | /*!40000 ALTER TABLE `lychee_photos` DISABLE KEYS */; 122 | /*!40000 ALTER TABLE `lychee_photos` ENABLE KEYS */; 123 | UNLOCK TABLES; 124 | 125 | -- 126 | -- Table structure for table `lychee_settings` 127 | -- 128 | 129 | DROP TABLE IF EXISTS `lychee_settings`; 130 | /*!40101 SET @saved_cs_client = @@character_set_client */; 131 | /*!40101 SET character_set_client = utf8 */; 132 | CREATE TABLE `lychee_settings` ( 133 | `key` varchar(50) NOT NULL DEFAULT '', 134 | `value` varchar(200) DEFAULT '' 135 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 136 | /*!40101 SET character_set_client = @saved_cs_client */; 137 | 138 | -- 139 | -- Dumping data for table `lychee_settings` 140 | -- 141 | -- login: lychee_ci / password: lychee_ci 142 | -- 143 | LOCK TABLES `lychee_settings` WRITE; 144 | /*!40000 ALTER TABLE `lychee_settings` DISABLE KEYS */; 145 | INSERT INTO `lychee_settings` VALUES ('version','030003'),('username','$2a$10$pmGfzUucdS2lMbXLAc5H5e.RkUHHGZFQIYSACrPpPnFXp.3kddlje'),('password','$2a$10$xY/P39dlcR3K5Eg7DL3kduL.BjKKDafceWwDPQa19zlrO2/buL1ua'),('thumbQuality','90'),('checkForUpdates','1'),('sortingPhotos','ORDER BY id DESC'),('sortingAlbums','ORDER BY id DESC'),('medium','1'),('imagick','1'),('dropboxKey',''),('identifier','e9a65d4887d9786ffb77d4b7108dd95a'),('skipDuplicates','0'),('plugins',''); 146 | /*!40000 ALTER TABLE `lychee_settings` ENABLE KEYS */; 147 | UNLOCK TABLES; 148 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 149 | 150 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 151 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 152 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 153 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 154 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 155 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 156 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 157 | 158 | -- Dump completed on 2016-01-09 18:31:19 159 | -------------------------------------------------------------------------------- /ressources/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | ; ignore pep8 errors on the following files 3 | pep8ignore = 4 | *.py E225 E501 5 | 6 | ; fix pep maxline too short for a modern display 7 | pep8maxlinelength = 120 8 | ; will ignor test in the following directory 9 | norecursedirs=venv* 10 | ; will use this fixture for all test 11 | ; don't know how to specifie multiples fixture (with different scope) in this 12 | usefixtures = carriagereturn 13 | -------------------------------------------------------------------------------- /ressources/test_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "db":"lychee_ci", 3 | "dbUser":"lychee_ci", 4 | "dbPassword":"lychee_ci", 5 | "dbHost":"localhost", 6 | "thumbQuality":80, 7 | "publicAlbum": 0, 8 | "excludeAlbums": [], 9 | "lycheepath": "/tmp/lychee", 10 | "testphotopath": "./tmptest", 11 | "testlib": "./tests/pics", 12 | "conf": "./ressources/test_conf.json" 13 | } 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | from setuptools import setup, find_packages 3 | 4 | from pip.download import PipSession 5 | from pip.req import parse_requirements 6 | 7 | reqs = [ 8 | str(r.req) for r in 9 | parse_requirements('requirements.txt', session=PipSession())] 10 | 11 | 12 | setup( 13 | name="lycheesync", 14 | version="3.0.9", 15 | author="Gustave Pate", 16 | author_email="gustave.pate@fake.com", 17 | description="Photo synchronization utility for Lychee", 18 | license="MIT", 19 | url="http://github.com/GustavePate/lycheesync", 20 | packages=find_packages(), 21 | classifiers=[ 22 | "Topic :: Utilities", 23 | "License :: OSI Approved :: MIT License", 24 | ], 25 | entry_points={ 26 | 'console_scripts': [ 27 | 'lycheesync=lycheesync.sync:main', 28 | ] 29 | }, 30 | install_requires=reqs, 31 | ) 32 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/__init__.py -------------------------------------------------------------------------------- /tests/configuration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pprint import pformat 5 | 6 | 7 | class Borg: 8 | _shared_state = {} 9 | 10 | def __init__(self): 11 | self.__dict__ = self._shared_state 12 | 13 | 14 | class TestBorg(Borg): 15 | 16 | isinitialized = False 17 | 18 | def __init__(self, confdic=None): 19 | Borg.__init__(self) 20 | if not (self.isinitialized): 21 | if confdic: 22 | self.confdic = confdic 23 | self.isinitialized = True 24 | else: 25 | raise Exception('ConfBorg not initialized') 26 | 27 | def __str__(self): 28 | return str(self.val) 29 | 30 | @property 31 | def pretty(self): 32 | return "\n" + pformat(self.confdic) 33 | 34 | @property 35 | def conf(self): 36 | return self.confdic 37 | -------------------------------------------------------------------------------- /tests/pics/FußÄ-Füße/Füße.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/FußÄ-Füße/Füße.jpg -------------------------------------------------------------------------------- /tests/pics/FußÄ-Füße/ok_ßüöä.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/FußÄ-Füße/ok_ßüöä.jpg -------------------------------------------------------------------------------- /tests/pics/aaa/Lychees---Nature_s-Pride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/aaa/Lychees---Nature_s-Pride.jpg -------------------------------------------------------------------------------- /tests/pics/aaa/Watercolor_Lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/aaa/Watercolor_Lychee.jpg -------------------------------------------------------------------------------- /tests/pics/aaa/fruit-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/aaa/fruit-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/aaa/lychee-fruit-21262197.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/aaa/lychee-fruit-21262197.jpg -------------------------------------------------------------------------------- /tests/pics/album1/large.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album1/large.1.jpg -------------------------------------------------------------------------------- /tests/pics/album2/album21/6640926-single-lychee-also-known-as-chinese-gooseberry-isolated-against-white-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album2/album21/6640926-single-lychee-also-known-as-chinese-gooseberry-isolated-against-white-background.jpg -------------------------------------------------------------------------------- /tests/pics/album2/album22/14828607-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album2/album22/14828607-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/album2/lychee_heart_1_by_yuri_chan1018-d4176jl.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album2/lychee_heart_1_by_yuri_chan1018-d4176jl.jpg -------------------------------------------------------------------------------- /tests/pics/album2/one-cut-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album2/one-cut-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/album3/Lychees---Nature_s-Pride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album3/Lychees---Nature_s-Pride.jpg -------------------------------------------------------------------------------- /tests/pics/album3/Watercolor_Lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album3/Watercolor_Lychee.jpg -------------------------------------------------------------------------------- /tests/pics/album3/fruit-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album3/fruit-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/album3/lychee-fruit-21262197.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/album3/lychee-fruit-21262197.jpg -------------------------------------------------------------------------------- /tests/pics/corrupted_file/Lychees---Nature_s-Pride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/corrupted_file/Lychees---Nature_s-Pride.jpg -------------------------------------------------------------------------------- /tests/pics/duplicates/large.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/duplicates/large.1.jpg -------------------------------------------------------------------------------- /tests/pics/duplicates/large.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/duplicates/large.2.jpg -------------------------------------------------------------------------------- /tests/pics/empty_album/tocommit.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/empty_album/tocommit.txt -------------------------------------------------------------------------------- /tests/pics/invalid_takedate/large.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/invalid_takedate/large.1.jpg -------------------------------------------------------------------------------- /tests/pics/invalid_taketime/IMG_0205.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/invalid_taketime/IMG_0205.JPG -------------------------------------------------------------------------------- /tests/pics/mini/P1060266.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/mini/P1060266.JPG -------------------------------------------------------------------------------- /tests/pics/real_date/fruit-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/real_date/fruit-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/real_date/fruit-lychee2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/real_date/fruit-lychee2.jpg -------------------------------------------------------------------------------- /tests/pics/rotation/P1010335.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/rotation/P1010335.JPG -------------------------------------------------------------------------------- /tests/pics/rotation/P1010336.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/rotation/P1010336.JPG -------------------------------------------------------------------------------- /tests/pics/rotation/P1010337.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/rotation/P1010337.JPG -------------------------------------------------------------------------------- /tests/pics/rotation/P1010338.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/rotation/P1010338.JPG -------------------------------------------------------------------------------- /tests/pics/with'"quotes/iu'"`stanragename.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/with'"quotes/iu'"`stanragename.jpg -------------------------------------------------------------------------------- /tests/pics/zzzz/Watercolor_Lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/zzzz/Watercolor_Lychee.jpg -------------------------------------------------------------------------------- /tests/pics/zzzz/fruit-lychee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/zzzz/fruit-lychee.jpg -------------------------------------------------------------------------------- /tests/pics/zzzz/lychee-fruit-21262197.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/pics/zzzz/lychee-fruit-21262197.jpg -------------------------------------------------------------------------------- /tests/standalone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GustavePate/lycheesync/043bc70d6f56ac791075bdc033b70d66f26ec60f/tests/standalone/__init__.py -------------------------------------------------------------------------------- /tests/standalone/db_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import pymysql 5 | 6 | 7 | def main(): 8 | 9 | db = pymysql.connect(host='127.0.0.1', 10 | user='lycheetest', 11 | passwd='lycheetest', 12 | db='lycheetest', 13 | charset='utf8mb4', 14 | unix_socket='/var/run/mysqld/mysqld.sock', 15 | cursorclass=pymysql.cursors.DictCursor) 16 | cur = db.cursor() 17 | cur.execute("set names utf8;") 18 | 19 | if __name__ == '__main__': 20 | main() 21 | -------------------------------------------------------------------------------- /tests/standalone/epoch_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dateutil.parser import parse 3 | import time 4 | 5 | 6 | def convert_strdate_to_timestamp(value): 7 | # check sysdate type 8 | print("convert_sysdate input: " + str(value)) 9 | print("convert_sysdate input_type: " + str(type(value))) 10 | 11 | timestamp = None 12 | # now in epoch time 13 | epoch_now = int(time.time()) 14 | 15 | if isinstance(value, int): 16 | timestamp = value 17 | elif isinstance(value, datetime.date): 18 | timestamp = (value - datetime.datetime(1970, 1, 1)).total_seconds() 19 | elif value: 20 | 21 | value = str(value) 22 | 23 | if isinstance(value, str): 24 | try: 25 | the_date = parse(value) 26 | print("DEBUG parsed date: " + str(the_date)) 27 | # woks for poython 3 28 | # timestamp = the_date.timestamp() 29 | timestamp = time.mktime(the_date.timetuple()) 30 | 31 | except Exception as e: 32 | print(e.message) 33 | print('WARN model sysdate impossible to parse: ' + str(value)) 34 | timestamp = epoch_now 35 | else: 36 | # Value is None 37 | timestamp = epoch_now 38 | 39 | return timestamp 40 | 41 | 42 | def main(): 43 | input = "2011-11-11 11:11:11" 44 | res = convert_strdate_to_timestamp(input) 45 | print('res: ' + str(res)) 46 | print('convert in: ' + str(datetime.datetime.fromtimestamp(res))) 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | import pytest 4 | import logging 5 | import subprocess 6 | import os 7 | import shutil 8 | import time 9 | import datetime 10 | from tests.testutils import TestUtils 11 | from click.testing import CliRunner 12 | from lycheesync.sync import main 13 | from PIL import Image 14 | import piexif 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | @pytest.mark.usefixtures("clean") 20 | @pytest.mark.usefixtures("initdb_and_fs") 21 | @pytest.mark.usefixtures("confborg") 22 | @pytest.mark.usefixtures("initloggers") 23 | class TestClass: 24 | 25 | def check_grand_total(self, expected_albums, expected_photos): 26 | 27 | tu = TestUtils() 28 | assert (tu.count_db_albums() == expected_albums), "there should be {} albums in db".format( 29 | expected_albums) 30 | assert (tu.count_db_photos() == expected_photos), "there should be {} photos in db".format( 31 | expected_photos) 32 | # assert Number of photo / thumbnail on filesystem 33 | # assert Number of photo / thumbnail on filesystem 34 | assert tu.count_fs_thumb() == expected_photos 35 | assert tu.count_fs_photos() == expected_photos 36 | 37 | def test_env_maker(self): 38 | tu = TestUtils() 39 | # clean all 40 | upload_path = os.path.join(tu.conf['lycheepath'], '/uploads') 41 | if os.path.exists(upload_path): 42 | shutil.rmtree(upload_path) 43 | tu.drop_db() 44 | tu.make_fake_lychee_db() 45 | tu.make_fake_lychee_fs(tu.conf['lycheepath']) 46 | # file system exists 47 | assert os.path.exists(os.path.join(tu.conf['lycheepath'], 'uploads', 'big')) 48 | assert os.path.exists(os.path.join(tu.conf['lycheepath'], 'uploads', 'medium')) 49 | assert os.path.exists(os.path.join(tu.conf['lycheepath'], 'uploads', 'thumb')) 50 | # table exists 51 | assert tu.table_exists('lychee_albums') 52 | assert tu.table_exists('lychee_photos') 53 | 54 | def test_subdir(self): 55 | tu = TestUtils() 56 | # copy directory to tmptest 57 | tu.load_photoset("album2") 58 | 59 | # launch lycheesync 60 | src = tu.conf['testphotopath'] 61 | lych = tu.conf['lycheepath'] 62 | conf = tu.conf['conf'] 63 | 64 | # run 65 | runner = CliRunner() 66 | result = runner.invoke(main, [src, lych, conf, '-v', '-d']) 67 | # no crash 68 | assert result.exit_code == 0, "process result is ok" 69 | 70 | # assert Number of album / photos in database 71 | expected_albums = 3 72 | expected_photos = 4 73 | assert (tu.count_db_albums() == expected_albums), "there should be {} albums in db".format( 74 | expected_albums) 75 | assert (tu.count_db_photos() == expected_photos), "there should be {} photos in db".format( 76 | expected_photos) 77 | # assert Number of photo / thumbnail on filesystem 78 | assert tu.count_fs_thumb() == expected_photos 79 | assert tu.count_fs_photos() == expected_photos 80 | 81 | def test_duplicate(self): 82 | tu = TestUtils() 83 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 84 | # copy directory to tmptest 85 | tu.load_photoset("album2") 86 | 87 | # launch lycheesync 88 | src = tu.conf['testphotopath'] 89 | lych = tu.conf['lycheepath'] 90 | conf = tu.conf['conf'] 91 | 92 | # run 93 | cmd = 'python main.py {} {} {} -v'.format(src, lych, conf) 94 | logger.info(cmd) 95 | retval = -1 96 | retval = subprocess.call(cmd, shell=True) 97 | # no crash 98 | assert (retval == 0), "process result is ok" 99 | 100 | # re-run 101 | cmd = 'python main.py {} {} {} -v'.format(src, lych, conf) 102 | logger.info(cmd) 103 | retval = -1 104 | retval = subprocess.call(cmd, shell=True) 105 | # no crash 106 | assert (retval == 0), "process result is ok" 107 | 108 | # assert Number of album / photos in database 109 | expected_albums = 3 110 | expected_photos = 4 111 | assert (tu.count_db_albums() == expected_albums), "there should be {} albums in db".format( 112 | expected_albums) 113 | assert (tu.count_db_photos() == expected_photos), "there should be {} photos in db".format( 114 | expected_photos) 115 | # assert Number of photo / thumbnail on filesystem 116 | # assert Number of photo / thumbnail on filesystem 117 | assert tu.count_fs_thumb() == expected_photos 118 | assert tu.count_fs_photos() == expected_photos 119 | 120 | def test_album_date(self): 121 | # album date should be equal to date/time original 122 | tu = TestUtils() 123 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 124 | # load album x and y 125 | tu.load_photoset("real_date") 126 | 127 | src = tu.conf['testphotopath'] 128 | lych = tu.conf['lycheepath'] 129 | conf = tu.conf['conf'] 130 | 131 | # run 132 | runner = CliRunner() 133 | result = runner.invoke(main, [src, lych, conf, '-v']) 134 | # no crash 135 | assert result.exit_code == 0, "process result is ok" 136 | 137 | assert tu.album_exists_in_db("real_date") 138 | # read album date for album1 139 | album1_date = tu.get_album_creation_date('real_date') 140 | 141 | real_date = datetime.datetime.fromtimestamp(album1_date) 142 | theorical_date = datetime.datetime(2011, 11, 11, 11, 11, 11) 143 | 144 | assert (real_date == theorical_date), "album date is 2011/11/11 11:11:11" 145 | 146 | def test_dash_r(self): 147 | try: 148 | tu = TestUtils() 149 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 150 | # load album x and y 151 | tu.load_photoset("album1") 152 | tu.load_photoset("album3") 153 | 154 | # launch lycheesync 155 | src = tu.conf['testphotopath'] 156 | lych = tu.conf['lycheepath'] 157 | conf = tu.conf['conf'] 158 | 159 | # run 160 | cmd = 'python main.py {} {} {} -r -v'.format(src, lych, conf) 161 | logger.info(cmd) 162 | retval = -1 163 | retval = subprocess.call(cmd, shell=True) 164 | # no crash 165 | assert (retval == 0), "process result is ok" 166 | 167 | assert tu.album_exists_in_db("album1") 168 | # read album date for album1 169 | album1_date = tu.get_album_creation_date('album1') 170 | # read album date for album3 171 | album3_date = tu.get_album_creation_date('album3') 172 | 173 | # empty tmp pictures folder 174 | tu.delete_dir_content(src) 175 | 176 | tu.dump_table('lychee_albums') 177 | # sleep 1 s to make time album signature different 178 | time.sleep(2) 179 | 180 | # load album3 181 | tu.load_photoset("album3") 182 | 183 | # run 184 | cmd = 'python main.py {} {} {} -r -v'.format(src, lych, conf) 185 | logger.info(cmd) 186 | retval = -1 187 | retval = subprocess.call(cmd, shell=True) 188 | # no crash 189 | assert (retval == 0), "process result is ok" 190 | 191 | album1_date_2 = tu.get_album_creation_date('album1') 192 | album3_date_2 = tu.get_album_creation_date('album3') 193 | tu.dump_table('lychee_albums') 194 | # y date < time 195 | assert album1_date == album1_date_2, 'album 1 is untouched' 196 | assert tu.check_album_size('album1') == 1 197 | 198 | # x date > time 199 | assert album3_date < album3_date_2, 'album 3 has been modified' 200 | assert tu.check_album_size('album3') == 4 201 | 202 | expected_albums = 2 203 | expected_photos = 5 204 | self.check_grand_total(expected_albums, expected_photos) 205 | 206 | except AssertionError: 207 | raise 208 | except Exception as e: 209 | logger.exception(e) 210 | assert False 211 | 212 | def test_dash_d(self): 213 | try: 214 | # load album y 215 | tu = TestUtils() 216 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 217 | # load album x and y 218 | tu.load_photoset("album1") 219 | # launch lycheesync 220 | src = tu.conf['testphotopath'] 221 | lych = tu.conf['lycheepath'] 222 | conf = tu.conf['conf'] 223 | 224 | # run 225 | cmd = 'python main.py {} {} {} -r -v'.format(src, lych, conf) 226 | logger.info(cmd) 227 | retval = -1 228 | retval = subprocess.call(cmd, shell=True) 229 | # no crash 230 | assert (retval == 0), "process result is ok" 231 | 232 | assert tu.check_album_size("album1") == 1, "album 1 not correctly loaded" 233 | 234 | # clean input pics content 235 | tu.delete_dir_content(src) 236 | # load album x 237 | tu.load_photoset("album3") 238 | 239 | # run 240 | cmd = 'python main.py {} {} {} -v -d'.format(src, lych, conf) 241 | logger.info(cmd) 242 | retval = -1 243 | retval = subprocess.call(cmd, shell=True) 244 | # no crash 245 | assert (retval == 0), "process result is ok" 246 | 247 | assert tu.check_album_size("album3") == 4, "album 3 not correctly loaded" 248 | # album 1 has been deleted 249 | a1_check = tu.album_exists_in_db("album1") 250 | assert not(a1_check), "album 1 still exists" 251 | 252 | expected_albums = 1 253 | expected_photos = 4 254 | self.check_grand_total(expected_albums, expected_photos) 255 | 256 | except AssertionError: 257 | raise 258 | except Exception as e: 259 | logger.exception(e) 260 | assert False 261 | 262 | # without -s albums id should not be sorted by name 263 | def test_dash_wo_s(self): 264 | # -s => no album reorder 265 | tu = TestUtils() 266 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 267 | # load a bunch of album 268 | tu.load_photoset("aaa") 269 | tu.load_photoset("mini") 270 | tu.load_photoset("zzzz") 271 | tu.load_photoset("album1") 272 | tu.load_photoset("album3") 273 | # launch lycheesync 274 | src = tu.conf['testphotopath'] 275 | lych = tu.conf['lycheepath'] 276 | conf = tu.conf['conf'] 277 | 278 | # run 279 | runner = CliRunner() 280 | result = runner.invoke(main, [src, lych, conf, '-v']) 281 | # no crash 282 | assert result.exit_code == 0, "process result is ok" 283 | 284 | # get a_id, a_names 285 | list = tu.get_album_ids_titles() 286 | logger.info(list) 287 | # album name sorted 288 | 289 | # id sorted 290 | ids = sorted([x['id'] for x in list]) 291 | titles = sorted([x['title'] for x in list]) 292 | 293 | logger.info(ids) 294 | logger.info(titles) 295 | 296 | # combine 297 | ordered_list = zip(ids, titles) 298 | logger.info(ordered_list) 299 | # for each sorted 300 | well_sorted = True 301 | for x in ordered_list: 302 | logger.info(x) 303 | 304 | if (tu.get_album_id(x[1]) != x[0]): 305 | well_sorted = False 306 | 307 | assert (not well_sorted), "elements should not be sorted" 308 | 309 | # with the -s switch album ids should be sorted by album title 310 | def test_dash_s(self): 311 | # -s => no album reorder 312 | tu = TestUtils() 313 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 314 | # load a bunch of album 315 | tu.load_photoset("aaa") 316 | tu.load_photoset("mini") 317 | tu.load_photoset("zzzz") 318 | tu.load_photoset("album1") 319 | tu.load_photoset("album3") 320 | # launch lycheesync 321 | src = tu.conf['testphotopath'] 322 | lych = tu.conf['lycheepath'] 323 | conf = tu.conf['conf'] 324 | 325 | # run 326 | runner = CliRunner() 327 | result = runner.invoke(main, [src, lych, conf, '-v', '-s']) 328 | # no crash 329 | assert result.exit_code == 0, "process result is ok" 330 | 331 | # get a_id, a_names 332 | list = tu.get_album_ids_titles() 333 | logger.info(list) 334 | # album name sorted 335 | 336 | # id sorted 337 | ids = sorted([x['id'] for x in list]) 338 | titles = sorted([x['title'] for x in list]) 339 | 340 | logger.info(ids) 341 | logger.info(titles) 342 | 343 | # combine 344 | ordered_list = zip(ids, titles) 345 | logger.info(ordered_list) 346 | # for each sorted 347 | for x in ordered_list: 348 | assert (tu.get_album_id(x[1]) == x[0]), "element not ordered " + x[1] 349 | 350 | def test_dash_l(self): 351 | # load album y 352 | tu = TestUtils() 353 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 354 | # load album x and y 355 | tu.load_photoset("album1") 356 | # launch lycheesync 357 | src = tu.conf['testphotopath'] 358 | lych = tu.conf['lycheepath'] 359 | conf = tu.conf['conf'] 360 | # -l => symbolic links instead of files 361 | 362 | # run 363 | runner = CliRunner() 364 | result = runner.invoke(main, [src, lych, conf, '-v', '-l']) 365 | # no crash 366 | assert result.exit_code == 0, "process result is ok" 367 | 368 | # check if files are links 369 | dest = os.path.join(lych, "uploads", "big") 370 | not_dir = [x for x in os.listdir(dest) if not(os.path.isdir(x))] 371 | for f in not_dir: 372 | full_path = os.path.join(dest, f) 373 | assert os.path.islink(full_path), "this file {} is not a link".format(full_path) 374 | 375 | def test_unicode(self): 376 | # there is a unicode album 377 | # there is a unicode photo 378 | tu = TestUtils() 379 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 380 | # load unicode album name 381 | tu.load_photoset("FußÄ-Füße") 382 | # launch lycheesync 383 | src = tu.conf['testphotopath'] 384 | lych = tu.conf['lycheepath'] 385 | conf = tu.conf['conf'] 386 | 387 | # run 388 | runner = CliRunner() 389 | result = runner.invoke(main, [src, lych, conf, '-v']) 390 | # no crash 391 | assert result.exit_code == 0, "process result is ok" 392 | 393 | assert tu.count_fs_photos() == 2, "photos are missing in fs" 394 | assert tu.count_db_photos() == 2, "photos are missing in db" 395 | assert tu.album_exists_in_db("FußÄ-Füße"), "unicode album is not in db" 396 | 397 | def test_corrupted(self): 398 | # load 1 album with a corrupted file 399 | tu = TestUtils() 400 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 401 | # load unicode album name 402 | tu.load_photoset("corrupted_file") 403 | # launch lycheesync 404 | src = tu.conf['testphotopath'] 405 | lych = tu.conf['lycheepath'] 406 | conf = tu.conf['conf'] 407 | 408 | # run 409 | runner = CliRunner() 410 | result = runner.invoke(main, [src, lych, conf, '-d', '-v']) 411 | # no crash 412 | assert result.exit_code == 0, "process result is ok" 413 | 414 | # no import 415 | assert tu.count_fs_photos() == 0, "there are photos in fs" 416 | assert tu.count_db_photos() == 0, "there are photos in db" 417 | assert tu.album_exists_in_db("corrupted_file"), "corrupted_album not in db" 418 | 419 | def test_empty_album(self): 420 | # load 1 empty album 421 | tu = TestUtils() 422 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 423 | # load unicode album name 424 | tu.load_photoset("empty_album") 425 | # launch lycheesync 426 | src = tu.conf['testphotopath'] 427 | lych = tu.conf['lycheepath'] 428 | conf = tu.conf['conf'] 429 | 430 | # run 431 | runner = CliRunner() 432 | result = runner.invoke(main, [src, lych, conf, '-v']) 433 | # no crash 434 | assert result.exit_code == 0, "process result is ok" 435 | 436 | # no import 437 | assert tu.count_fs_photos() == 0, "there are photos are in fs" 438 | assert tu.count_db_photos() == 0, "there are photos are in db" 439 | assert not(tu.album_exists_in_db("empty_album")), "empty_album in db" 440 | 441 | def test_long_album(self): 442 | tu = TestUtils() 443 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 444 | # get max_width column album name width 445 | maxwidth = tu.get_column_width("lychee_albums", "title") 446 | logger.info("album title length: " + str(maxwidth)) 447 | # create long album name 448 | dest_alb_name = 'a' * (maxwidth + 10) 449 | assert len(dest_alb_name) == (maxwidth + 10) 450 | 451 | # copy album with name 452 | tu.load_photoset("album1", dest_alb_name) 453 | 454 | # launch lycheesync 455 | src = tu.conf['testphotopath'] 456 | lych = tu.conf['lycheepath'] 457 | conf = tu.conf['conf'] 458 | 459 | # run 460 | runner = CliRunner() 461 | result = runner.invoke(main, [src, lych, conf, '-v']) 462 | # no crash 463 | assert result.exit_code == 0, "process result is ok" 464 | 465 | # there is a max_width album 466 | albums = tu.get_album_ids_titles() 467 | alb_real_name = albums.pop()["title"] 468 | assert len(alb_real_name) == maxwidth, "album len is not " + str(maxwidth) 469 | 470 | def test_sha1(self): 471 | """ 472 | Should also trigger a warn 473 | duplicates containes photos from album1 474 | """ 475 | tu = TestUtils() 476 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 477 | # load 1 album with same photo under different name 478 | tu.load_photoset("album1") 479 | # load 2 album with same photo under different name 480 | tu.load_photoset("duplicates") 481 | 482 | # launch lycheesync 483 | src = tu.conf['testphotopath'] 484 | lych = tu.conf['lycheepath'] 485 | conf = tu.conf['conf'] 486 | 487 | # run 488 | runner = CliRunner() 489 | result = runner.invoke(main, [src, lych, conf, '-v']) 490 | # no crash 491 | assert result.exit_code == 0, "process result is ok" 492 | 493 | # no duplicate 494 | assert tu.count_db_albums() == 2, "two albums not created" 495 | assert tu.count_fs_photos() == 2, "there are duplicate photos in fs" 496 | assert tu.count_db_photos() == 2, "there are duplicate photos in db" 497 | assert tu.count_fs_thumb() == 2, "there are duplicate photos in thumb" 498 | 499 | def test_album_keep_original_case(self): 500 | # load 1 album with a mixed case name and spaces 501 | # name in db is equal to directory name 502 | tu = TestUtils() 503 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 504 | # load 1 album with same photo under different name 505 | tu.load_photoset("album1", "AlBum_One") 506 | 507 | # launch lycheesync 508 | src = tu.conf['testphotopath'] 509 | lych = tu.conf['lycheepath'] 510 | conf = tu.conf['conf'] 511 | 512 | # run 513 | runner = CliRunner() 514 | result = runner.invoke(main, [src, lych, conf, '-v']) 515 | # no crash 516 | assert result.exit_code == 0, "process result is ok" 517 | 518 | assert tu.count_db_albums() == 1, "two albums created" 519 | assert tu.count_fs_photos() == 1, "there are duplicate photos in fs" 520 | assert tu.count_db_photos() == 1, "there are duplicate photos in db" 521 | assert tu.count_fs_thumb() == 1, "there are duplicate photos in thumb" 522 | assert tu.get_album_id("AlBum_One"), 'there is no album with this name' 523 | 524 | def test_bad_taketime(self): 525 | # load "bad taketime" album name 526 | tu = TestUtils() 527 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 528 | # load 1 album with same photo under different name 529 | tu.load_photoset("invalid_takedate") 530 | launch_date = datetime.datetime.now() 531 | time.sleep(1) 532 | # launch lycheesync 533 | src = tu.conf['testphotopath'] 534 | lych = tu.conf['lycheepath'] 535 | conf = tu.conf['conf'] 536 | 537 | # run 538 | runner = CliRunner() 539 | result = runner.invoke(main, [src, lych, conf, '-v']) 540 | # no crash 541 | assert result.exit_code == 0, "process result is ok" 542 | 543 | assert tu.count_db_albums() == 1, "two albums created" 544 | assert tu.count_fs_photos() == 1, "there are duplicate photos in fs" 545 | assert tu.count_db_photos() == 1, "there are duplicate photos in db" 546 | assert tu.count_fs_thumb() == 1, "there are duplicate photos in thumb" 547 | creation_date = tu.get_album_creation_date("invalid_takedate") 548 | creation_date = datetime.datetime.fromtimestamp(creation_date) 549 | assert creation_date > launch_date, "creation date should be now" 550 | 551 | def test_invalid_taketime(self): 552 | # load "bad taketime" album name 553 | tu = TestUtils() 554 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 555 | # load 1 album with same photo under different name 556 | tu.load_photoset("invalid_taketime") 557 | 558 | src = tu.conf['testphotopath'] 559 | lych = tu.conf['lycheepath'] 560 | conf = tu.conf['conf'] 561 | 562 | # run 563 | runner = CliRunner() 564 | result = runner.invoke(main, [src, lych, conf, '-v']) 565 | # no crash 566 | assert result.exit_code == 0, "process result is ok" 567 | 568 | assert tu.count_db_albums() == 1, "too much albums created" 569 | assert tu.count_fs_photos() == 1, "there are duplicate photos in fs" 570 | assert tu.count_db_photos() == 1, "there are duplicate photos in db" 571 | assert tu.count_fs_thumb() == 1, "there are duplicate photos in thumb" 572 | 573 | def test_quotes_in_album_name(self): 574 | # load "bad taketime" album name 575 | tu = TestUtils() 576 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 577 | # load 1 album with same photo under different name 578 | tu.load_photoset("with'\"quotes") 579 | 580 | src = tu.conf['testphotopath'] 581 | lych = tu.conf['lycheepath'] 582 | conf = tu.conf['conf'] 583 | 584 | # run 585 | runner = CliRunner() 586 | result = runner.invoke(main, [src, lych, conf, '-v']) 587 | # no crash 588 | assert result.exit_code == 0, "process result is ok" 589 | 590 | assert tu.count_db_albums() == 1, "too much albums created" 591 | assert tu.count_fs_photos() == 1, "there are duplicate photos in fs" 592 | assert tu.count_db_photos() == 1, "there are duplicate photos in db" 593 | assert tu.count_fs_thumb() == 1, "there are duplicate photos in thumb" 594 | 595 | def test_photoid_equal_timestamp(self): 596 | tu = TestUtils() 597 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 598 | # load 1 album with same photo under different name 599 | tu.load_photoset("album3") 600 | # launch lycheesync 601 | src = tu.conf['testphotopath'] 602 | lych = tu.conf['lycheepath'] 603 | conf = tu.conf['conf'] 604 | # normal mode 605 | before_launch = datetime.datetime.now() 606 | time.sleep(1.1) 607 | 608 | # run 609 | runner = CliRunner() 610 | result = runner.invoke(main, [src, lych, conf, '-v']) 611 | # no crash 612 | assert result.exit_code == 0, "process result is ok" 613 | 614 | time.sleep(1.1) 615 | after_launch = datetime.datetime.now() 616 | photos = tu.get_photos(tu.get_album_id('album3')) 617 | for p in photos: 618 | logger.info(p) 619 | # substract 4 last characters 620 | ts = str(p['id'])[:-4] 621 | 622 | # timestamp to date 623 | dt = datetime.datetime.fromtimestamp(int(ts)) 624 | logger.info(dt) 625 | assert after_launch > dt, "date from id not < date after launch" 626 | assert dt > before_launch, "date from id not > date before launch" 627 | 628 | def test_shutter_speed(self): 629 | tu = TestUtils() 630 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 631 | # load 1 album with same photo under different name 632 | tu.load_photoset("rotation") 633 | # launch lycheesync 634 | src = tu.conf['testphotopath'] 635 | lych = tu.conf['lycheepath'] 636 | conf = tu.conf['conf'] 637 | 638 | # run 639 | runner = CliRunner() 640 | result = runner.invoke(main, [src, lych, conf, '-v']) 641 | # no crash 642 | assert result.exit_code == 0, "process result is ok" 643 | 644 | photos = tu.get_photos(tu.get_album_id('rotation')) 645 | for p in photos: 646 | if p['title'] == 'P1010319.JPG': 647 | assert p['shutter'] == '1/60 s', "shutter {} not equal 1/60 s".format(p['shutter']) 648 | assert p['focal'] == '4.9 mm', "focal {} not equal 4.9 mm".format(p['focal']) 649 | assert p['iso'] == '100', "iso {} not equal 100".format(p['iso']) 650 | assert p['aperture'] == 'F3.3', "aperture {} not equal F3.3".format(p['aperture']) 651 | if p['title'] == 'P1010328.JPG': 652 | assert p['shutter'] == '1/30 s', "shutter {} not equal 1/30 s".format(p['shutter']) 653 | assert p['focal'] == '4.9 mm', "focal {} not equal 4.9 mm".format(p['focal']) 654 | assert p['iso'] == '400', "iso {} not equal 400".format(p['iso']) 655 | assert p['aperture'] == 'F3.3', "aperture {} not equal F3.3".format(p['aperture']) 656 | 657 | def test_rotation(self): 658 | 659 | tu = TestUtils() 660 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 661 | # load 1 album with same photo under different name 662 | tu.load_photoset("rotation") 663 | # launch lycheesync 664 | src = tu.conf['testphotopath'] 665 | lych = tu.conf['lycheepath'] 666 | conf = tu.conf['conf'] 667 | 668 | # run 669 | runner = CliRunner() 670 | result = runner.invoke(main, [src, lych, conf, '-v']) 671 | # no crash 672 | assert result.exit_code == 0, "process result is ok" 673 | 674 | photos = tu.get_photos(tu.get_album_id('rotation')) 675 | for p in photos: 676 | # rotation tag is gone 677 | pfullpath = os.path.join(lych, "uploads", "big", p['url']) 678 | img = Image.open(pfullpath) 679 | assert "exif" in img.info, "Pas d'info exif" 680 | exif_dict = piexif.load(img.info["exif"]) 681 | assert exif_dict["0th"][piexif.ImageIFD.Orientation] == 1, "Exif rotation should be 1" 682 | img.close() 683 | 684 | def test_launch_every_test_with_cli_runner(self): 685 | """ conf borg is shared between test and cli, this is potentially bad""" 686 | try: 687 | # load "bad taketime" album name 688 | tu = TestUtils() 689 | tu.load_photoset("album3") 690 | # launch lycheesync 691 | src = tu.conf['testphotopath'] 692 | lych = tu.conf['lycheepath'] 693 | conf = tu.conf['conf'] 694 | # run 695 | runner = CliRunner() 696 | result = runner.invoke(main, [src, lych, conf, '-v']) 697 | # no crash 698 | assert result.exit_code == 0, "process result is ok" 699 | except Exception as e: 700 | logger.exception(e) 701 | assert False 702 | 703 | def test_launch_with_wo_clirunner_w_main_py(self): 704 | tu = TestUtils() 705 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 706 | # load 1 album with same photo under different name 707 | tu.load_photoset("album3") 708 | # launch lycheesync 709 | src = tu.conf['testphotopath'] 710 | lych = tu.conf['lycheepath'] 711 | conf = tu.conf['conf'] 712 | 713 | # run 714 | cmd = 'python main.py {} {} {} -v'.format(src, lych, conf) 715 | logger.info(cmd) 716 | retval = -1 717 | retval = subprocess.call(cmd, shell=True) 718 | # no crash 719 | assert (retval == 0), "process result is ok" 720 | 721 | assert tu.count_db_albums() == 1, "too much albums created" 722 | assert tu.count_fs_photos() == 4, "there are duplicate photos in fs" 723 | assert tu.count_db_photos() == 4, "there are duplicate photos in db" 724 | assert tu.count_fs_thumb() == 4, "there are duplicate photos in thumb" 725 | 726 | def test_launch_with_wo_clirunner_w_sync_py(self): 727 | tu = TestUtils() 728 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 729 | # load 1 album with same photo under different name 730 | tu.load_photoset("album3") 731 | # launch lycheesync 732 | src = tu.conf['testphotopath'] 733 | lych = tu.conf['lycheepath'] 734 | conf = tu.conf['conf'] 735 | 736 | # run 737 | cmd = 'python -m lycheesync.sync {} {} {} -v'.format(src, lych, conf) 738 | logger.info(cmd) 739 | retval = -1 740 | retval = subprocess.call(cmd, shell=True) 741 | # no crash 742 | assert (retval == 0), "process result is ok" 743 | 744 | assert tu.count_db_albums() == 1, "too much albums created" 745 | assert tu.count_fs_photos() == 4, "there are duplicate photos in fs" 746 | assert tu.count_db_photos() == 4, "there are duplicate photos in db" 747 | assert tu.count_fs_thumb() == 4, "there are duplicate photos in thumb" 748 | 749 | def test_sanity(self): 750 | # load album 751 | tu = TestUtils() 752 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 753 | # launch lycheesync 754 | src = tu.conf['testphotopath'] 755 | lych = tu.conf['lycheepath'] 756 | conf = tu.conf['conf'] 757 | lib = tu.conf['testlib'] 758 | 759 | # album will be remove 760 | album3_path = os.path.join(lib, "album3") 761 | album_3_copy = os.path.join(lib, "album3_tmp") 762 | if os.path.isdir(album_3_copy): 763 | shutil.rmtree(album_3_copy) 764 | shutil.copytree(album3_path, album_3_copy) 765 | tu.load_photoset("album3_tmp") 766 | # insert garbage files 767 | lychee_photo_path = os.path.join(lych, "uploads", "big") 768 | 769 | lib_photo_1 = os.path.join(lib, "album1", "large.1.jpg") 770 | lib_photo_2 = os.path.join(lib, "real_date", "fruit-lychee.jpg") 771 | lib_photo_3 = os.path.join(lib, "real_date", "fruit-lychee2.jpg") 772 | lib_photo_4 = os.path.join(lib, "album2", "one-cut-lychee.jpg") 773 | a_photo_1 = os.path.join(lychee_photo_path, "large.1.jpg") 774 | a_photo_2 = os.path.join(lychee_photo_path, "link_src.jpg") 775 | a_photo_3 = os.path.join(lychee_photo_path, "broken_link_src.jpg") 776 | a_photo_4 = os.path.join(lychee_photo_path, "orphan_in_db.jpg") 777 | a_link_1 = os.path.join(lychee_photo_path, "link_1.jpg") 778 | a_link_2 = os.path.join(lychee_photo_path, "link_2.jpg") 779 | a_link_3 = os.path.join(lychee_photo_path, "broken_link_3.jpg") 780 | 781 | # FS orphan photo 782 | shutil.copy(lib_photo_1, a_photo_1) 783 | shutil.copy(lib_photo_2, a_photo_2) 784 | shutil.copy(lib_photo_3, a_photo_3) 785 | shutil.copy(lib_photo_4, a_photo_4) 786 | # FS orphan os.link 787 | os.link(a_photo_2, a_link_1) 788 | # FS orphan os.symlink 789 | os.symlink(a_photo_2, a_link_2) 790 | # FS orphan broken link 791 | os.symlink(a_photo_3, a_link_3) 792 | os.remove(a_photo_3) 793 | assert os.path.islink(a_link_3), "{} should be a link".format(a_link_3) 794 | assert not(os.path.exists(a_link_3)), "{} should be a broken link".format(a_link_3) 795 | try: 796 | db = tu._connect_db() 797 | 798 | # DB empty album in db 799 | tu._exec_sql( 800 | db, 801 | "insert into lychee_albums (id, title, sysstamp, public, visible, downloadable) values (25, 'orphan', 123, 1, 1 ,1)") 802 | 803 | # DB orphan photo in db 804 | tu._exec_sql( 805 | db, 806 | "insert into lychee_photos (id, title, url, tags, public, type, width, height, size, iso, aperture, model, shutter, focal, star, thumbUrl,album, medium) values (2525, 'orphan', 'one-cut-lychee.jpg', '', 1, 'jpg', 500, 500, '2323px', '100', 'F5.5', 'FZ5', '1s', 'F2', 0, 'thumburl.jpg', 666, 0)") 807 | 808 | finally: 809 | db.close() 810 | 811 | # test if well created 812 | assert tu.photo_exists_in_db(2525), "orphan photo exists" 813 | assert tu.album_exists_in_db('orphan'), "orphan album should exists" 814 | 815 | # launch with link and sanity option 816 | runner = CliRunner() 817 | result = runner.invoke(main, [src, lych, conf, '-v', '-l', '--sanitycheck']) 818 | # no crash 819 | assert result.exit_code == 0, "process result is ok" 820 | 821 | # garbage is gone 822 | 823 | # DB 824 | 825 | # no empty album 826 | assert not tu.album_exists_in_db('orphan'), "orphan album should have been deleted" 827 | 828 | # no orphan photo 829 | assert not tu.photo_exists_in_db(2525), "orphan photo should have been deleted" 830 | 831 | # FS 832 | 833 | # no broken symlink 834 | assert not(os.path.lexists(a_link_3)), "{} should have been deleted as a broken link".format(a_link_3) 835 | assert not(os.path.islink(a_link_3)), "{} should have been deleted".format(a_link_3) 836 | 837 | # no orphan link 838 | assert not(os.path.lexists(a_link_1)), "{} should have been deleted as a broken link".format(a_link_1) 839 | assert not(os.path.lexists(a_link_2)), "{} should have been deleted as a broken link".format(a_link_2) 840 | 841 | # no orphan photo 842 | assert not(os.path.exists(a_photo_1)), "{} should have been deleted as a orphan".format(a_photo_1) 843 | assert not(os.path.exists(a_photo_2)), "{} should have been deleted as an orphan".format(a_photo_2) 844 | assert not(os.path.exists(a_photo_3)), "{} should have been deleted as an orphan".format(a_photo_3) 845 | assert not(os.path.exists(a_photo_4)), "{} should have been deleted as an orphan".format(a_photo_4) 846 | 847 | def test_visually_check_logs(self): 848 | # load "bad taketime" album name 849 | tu = TestUtils() 850 | assert tu.is_env_clean(tu.conf['lycheepath']), "env not clean" 851 | # load 1 album with same photo under different name 852 | tu.load_photoset("invalid_takedate") 853 | tu.load_photoset("album2") 854 | tu.load_photoset("album3") 855 | tu.load_photoset("corrupted_file") 856 | tu.load_photoset("duplicates") 857 | # launch lycheesync 858 | src = tu.conf['testphotopath'] 859 | lych = tu.conf['lycheepath'] 860 | conf = tu.conf['conf'] 861 | 862 | # run 863 | runner = CliRunner() 864 | result = runner.invoke(main, [src, lych, conf, '-v']) 865 | # no crash 866 | assert result.exit_code == 0, "process result is ok" 867 | 868 | assert tu.count_db_albums() == 7, "too much albums created" 869 | assert tu.count_fs_photos() == 10, "there are duplicate photos in fs" 870 | assert tu.count_db_photos() == 10, "there are duplicate photos in db" 871 | assert tu.count_fs_thumb() == 10, "there are duplicate photos in thumb" 872 | -------------------------------------------------------------------------------- /tests/testutils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | import logging 4 | import os 5 | import glob 6 | import shutil 7 | import re 8 | import subprocess 9 | import pymysql 10 | import base64 11 | from tests.configuration import TestBorg 12 | from lycheesync.utils.configuration import ConfBorg 13 | # from datetime import datetime 14 | import pymysql.cursors 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class TestUtils: 20 | 21 | def __init__(self): 22 | self.db = None 23 | self.cb = TestBorg() 24 | self._lychee_sync_conf_borg = ConfBorg(force_init=True) 25 | 26 | @property 27 | def conf(self): 28 | return self.cb.conf 29 | 30 | def _connect_db(self): 31 | db = pymysql.connect(host=self.cb.conf['dbHost'], 32 | user=self.cb.conf['dbUser'], 33 | passwd=self.cb.conf['dbPassword'], 34 | db=self.cb.conf['db'], 35 | charset='utf8mb4', 36 | cursorclass=pymysql.cursors.DictCursor) 37 | return db 38 | 39 | def _exec_sql(self, db, sql): 40 | try: 41 | with db.cursor() as cursor: 42 | cursor.execute(sql) 43 | 44 | db.commit() 45 | 46 | except Exception as e: 47 | raise e 48 | 49 | def make_fake_lychee_fs(self, path): 50 | 51 | if not(os.path.isdir(os.path.join(path, 'uploads', 'big'))): 52 | uploads = os.path.join(path, 'uploads') 53 | os.mkdir(path) 54 | os.mkdir(uploads) 55 | paths = [] 56 | paths.append(os.path.join(uploads, 'big')) 57 | paths.append(os.path.join(uploads, 'thumb')) 58 | paths.append(os.path.join(uploads, 'medium')) 59 | for p in paths: 60 | if not(os.path.isdir(p)): 61 | logger.info('mkdir ' + p) 62 | os.mkdir(p) 63 | 64 | def is_env_clean(self, path): 65 | check = [] 66 | try: 67 | folders = {} 68 | folders['big'] = os.path.join(path, 'uploads', 'big') 69 | folders['medium'] = os.path.join(path, 'uploads', 'medium') 70 | folders['thumb'] = os.path.join(path, 'uploads', 'thumb') 71 | # is dir 72 | check.append(os.path.isdir(folders['big'])) 73 | check.append(os.path.isdir(folders['medium'])) 74 | check.append(os.path.isdir(folders['thumb'])) 75 | # is empty 76 | path = folders['big'] 77 | check.append(0 == len([f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))])) 78 | 79 | subprocess.call("ls /tmp/lychee/uploads/big/ -la", shell=True) 80 | assert len([f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]) == 0, "big is not empty" 81 | path = folders['medium'] 82 | check.append(0 == len([f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))])) 83 | path = folders['thumb'] 84 | check.append(0 == len([f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))])) 85 | # count album == 0 86 | albums = self.get_album_ids_titles() 87 | check.append(len(albums) == 0) 88 | 89 | # count photos == 0 90 | check.append(len(self.get_photos()) == 0) 91 | 92 | except Exception as e: 93 | logger.exception(e) 94 | check.append(False) 95 | finally: 96 | res = True 97 | for c in check: 98 | if not(c): 99 | res = False 100 | return res 101 | 102 | def drop_db(self): 103 | # connect to db 104 | if self.db_exists(): 105 | try: 106 | self.db = pymysql.connect(host=self.cb.conf['dbHost'], 107 | user=self.cb.conf['dbUser'], 108 | passwd=self.cb.conf['dbPassword'], 109 | charset='utf8mb4', 110 | cursorclass=pymysql.cursors.DictCursor) 111 | # check if db exists 112 | sql = "DROP DATABASE " + self.cb.conf['db'] 113 | with self.db.cursor() as cursor: 114 | cursor.execute(sql) 115 | self.db.commit() 116 | except Exception as e: 117 | logger.exception(e) 118 | finally: 119 | self.db.close() 120 | 121 | def table_exists(self, table_name): 122 | res = False 123 | db = self._connect_db() 124 | try: 125 | sql = "show tables where Tables_in_{}='lychee_albums';".format(self.cb.conf['db']) 126 | with db.cursor() as cursor: 127 | cursor.execute(sql) 128 | res = (len(cursor.fetchall()) == 1) 129 | except Exception as e: 130 | raise e 131 | finally: 132 | db.close() 133 | return res 134 | 135 | def db_exists(self): 136 | res = True 137 | try: 138 | db = self._connect_db() 139 | except Exception as e: 140 | logger.warn("db does not exist yet... %s", e) 141 | res = False 142 | finally: 143 | try: 144 | db.close() 145 | finally: 146 | return res 147 | 148 | def make_fake_lychee_db(self): 149 | 150 | # connect to db 151 | self.db = pymysql.connect(host=self.cb.conf['dbHost'], 152 | user=self.cb.conf['dbUser'], 153 | passwd=self.cb.conf['dbPassword'], 154 | charset='utf8mb4', 155 | cursorclass=pymysql.cursors.DictCursor) 156 | # check if db exists 157 | try: 158 | sql = "CREATE DATABASE IF NOT EXISTS " + self.cb.conf['db'] 159 | with self.db.cursor() as cursor: 160 | cursor.execute(sql) 161 | self.db.commit() 162 | except Exception as e: 163 | logger.exception(e) 164 | finally: 165 | self.db.close() 166 | 167 | db = self._connect_db() 168 | 169 | try: 170 | # check if table exists 171 | sql = "show tables where Tables_in_{}='lychee_albums';".format(self.cb.conf['db']) 172 | with db.cursor() as cursor: 173 | cursor.execute(sql) 174 | 175 | if (len(cursor.fetchall()) == 0): 176 | cmd = 'mysql -h {} -u {} -p{} {} < ./ressources/lychee.sql'.format( 177 | self.cb.conf['dbHost'], 178 | self.cb.conf['dbUser'], 179 | self.cb.conf['dbPassword'], 180 | self.cb.conf['db']) 181 | logger.info(cmd) 182 | retval = -1 183 | retval = subprocess.call(cmd, shell=True) 184 | assert retval == 0 185 | 186 | # create tables 187 | self.init_db_settings() 188 | except Exception as e: 189 | logger.exception(e) 190 | finally: 191 | db.close() 192 | 193 | def init_db_settings(self): 194 | db = self._connect_db() 195 | try: 196 | username = "foo" 197 | password = "bar" 198 | b64_username = base64.b64encode(username) 199 | b64_password = base64.b64encode(password) 200 | qry = "insert into lychee_settings (`key`, `value`) values ('username', %s)" 201 | with db.cursor() as cursor: 202 | try: 203 | cursor.execute(qry, (b64_username)) 204 | except Exception as e: 205 | logger.warn(e) 206 | logger.warn(cursor) 207 | qry = "insert into lychee_settings (`key`, `value`) values ('password', %s)" 208 | with db.cursor() as cursor: 209 | cursor.execute(qry, (b64_password)) 210 | db.commit() 211 | except Exception as e: 212 | logger.exception(e) 213 | finally: 214 | db.close() 215 | 216 | def load_photoset(self, set_name, dest_name=None): 217 | testlibpath = self.cb.conf['testlib'] 218 | testalbum = os.path.join(testlibpath, set_name) 219 | 220 | if dest_name is None: 221 | dest_name = set_name 222 | 223 | dest = os.path.join(self.cb.conf['testphotopath'], dest_name) 224 | shutil.copytree(testalbum, dest) # , copy_function=shutil.copy) 225 | 226 | def clean_db(self): 227 | logger.info("Clean Database") 228 | # connect to db 229 | db = self._connect_db() 230 | try: 231 | if self.table_exists('lychee_albums'): 232 | self._exec_sql(db, "TRUNCATE TABLE lychee_albums;") 233 | if self.table_exists('lychee_photos'): 234 | self._exec_sql(db, "TRUNCATE TABLE lychee_photos;") 235 | except Exception as e: 236 | logger.exception(e) 237 | finally: 238 | try: 239 | db.close() 240 | finally: 241 | pass 242 | 243 | def _empty_or_create_dir(self, path): 244 | try: 245 | if os.path.isdir(path): 246 | path = os.path.join(path, '*') 247 | for f in glob.glob(path): 248 | if os.path.lexists(f): 249 | if os.path.isfile(f): 250 | if not f.endswith("html"): 251 | os.remove(f) 252 | elif os.path.isdir(f): 253 | shutil.rmtree(f) 254 | elif os.path.islink(f): 255 | os.unlink(f) 256 | else: 257 | logger.warn("will not remove: %s", f) 258 | else: 259 | logger.warn("will not remove not exists: %s", f) 260 | 261 | else: 262 | logger.info(path + ' not a dir, create it') 263 | os.mkdir(path) 264 | except Exception as e: 265 | logger.exception(e) 266 | 267 | def clean_fs(self): 268 | 269 | logger.info("***************Clean Filesystem") 270 | lycheepath = self.cb.conf['lycheepath'] 271 | # empty tmp directory 272 | tmpdir = self.cb.conf['testphotopath'] 273 | if os.path.isdir(tmpdir): 274 | shutil.rmtree(tmpdir) 275 | 276 | os.mkdir(tmpdir) 277 | 278 | # empty images 279 | big = os.path.join(lycheepath, "uploads", "big") 280 | med = os.path.join(lycheepath, "uploads", "medium") 281 | thumb = os.path.join(lycheepath, "uploads", "thumb") 282 | 283 | # empty images 284 | self._empty_or_create_dir(big) 285 | self._empty_or_create_dir(med) 286 | self._empty_or_create_dir(thumb) 287 | 288 | def delete_dir_content(self, dir): 289 | self._empty_or_create_dir(dir) 290 | 291 | def count_db_photos(self): 292 | res = -1 293 | db = self._connect_db() 294 | try: 295 | sql = "select count(1) as total from lychee_photos" 296 | with db.cursor() as cursor: 297 | cursor.execute(sql) 298 | count = cursor.fetchone() 299 | res = count['total'] 300 | except Exception as e: 301 | logger.exception(e) 302 | assert False 303 | finally: 304 | db.close() 305 | return res 306 | 307 | def dump_table(self, table_name): 308 | db = self._connect_db() 309 | try: 310 | sql = "select * from " + table_name 311 | with db.cursor() as cursor: 312 | cursor.execute(sql) 313 | rows = cursor.fetchall() 314 | for row in rows: 315 | logger.info(row) 316 | 317 | except Exception as e: 318 | logger.exception(e) 319 | assert False 320 | finally: 321 | db.close() 322 | 323 | def count_db_albums(self): 324 | res = -1 325 | db = self._connect_db() 326 | try: 327 | sql = "select count(1) as total from lychee_albums" 328 | with db.cursor() as cursor: 329 | cursor.execute(sql) 330 | count = cursor.fetchone() 331 | res = count['total'] 332 | except Exception as e: 333 | logger.exception(e) 334 | assert False 335 | finally: 336 | db.close() 337 | return res 338 | 339 | def get_album_creation_date(self, a_name): 340 | res = -1 341 | db = self._connect_db() 342 | try: 343 | sql = "select sysstamp from lychee_albums where title='{}'".format(a_name) 344 | with db.cursor() as cursor: 345 | cursor.execute(sql) 346 | data = cursor.fetchone() 347 | sysstamp = data['sysstamp'] 348 | # res = datetime.fromtimestamp(sysstamp) 349 | res = sysstamp 350 | except Exception as e: 351 | logger.exception(e) 352 | assert False 353 | finally: 354 | db.close() 355 | return res 356 | 357 | def _count_files_in_dir(self, path): 358 | res = -1 359 | try: 360 | res = len([name for name in os.listdir(path) if os.path.isfile( 361 | os.path.join(path, name)) or os.path.islink(os.path.join(path, name))]) 362 | # don't count index.html 363 | path = os.path.join(path, 'index.html') 364 | if os.path.isfile(path): 365 | res = res - 1 366 | except Exception as e: 367 | logger.exception(e) 368 | assert False 369 | finally: 370 | return res 371 | 372 | def count_fs_photos(self): 373 | path = os.path.join(self.cb.conf['lycheepath'], 'uploads', 'big') 374 | res = self._count_files_in_dir(path) 375 | return res 376 | 377 | def count_fs_thumb(self): 378 | path = os.path.join(self.cb.conf['lycheepath'], 'uploads', 'thumb') 379 | res = self._count_files_in_dir(path) 380 | # 2 thumbs per image 381 | return (res / 2) 382 | 383 | def album_exists_in_db(self, a_name): 384 | return (self.get_album_id(a_name) is not None) 385 | 386 | def get_album_id(self, a_name): 387 | res = None 388 | db = self._connect_db() 389 | try: 390 | # check if exists in db 391 | sql = "select id from lychee_albums where title='{}'".format(a_name) 392 | with db.cursor() as cursor: 393 | cursor.execute(sql) 394 | rows = cursor.fetchmany(size=2) 395 | if (len(rows) == 1 and rows[0]): 396 | res = rows[0]['id'] 397 | except Exception as e: 398 | logger.exception(e) 399 | res = None 400 | finally: 401 | db.close() 402 | return res 403 | 404 | def get_photos(self, a_id=None, p_id=None): 405 | """ get photos as a list of dictionnary. optionnal: a_id to get photos of one album """ 406 | res = None 407 | db = self._connect_db() 408 | try: 409 | # check if exists in db 410 | if a_id: 411 | sql = "select id, title, url, iso, aperture, shutter, focal from lychee_photos where album='{}'".format(a_id) 412 | elif p_id: 413 | sql = "select id, title, url, iso, aperture, shutter, focal from lychee_photos where id='{}'".format(p_id) 414 | else: 415 | sql = "select id, title, url, iso, aperture, shutter, focal from lychee_photos" 416 | 417 | with db.cursor() as cursor: 418 | cursor.execute(sql) 419 | rows = cursor.fetchall() 420 | res = [] 421 | for r in rows: 422 | photo = {} 423 | photo['url'] = r['url'] 424 | photo['id'] = r['id'] 425 | photo['iso'] = r['iso'] 426 | photo['aperture'] = r['aperture'] 427 | photo['shutter'] = r['shutter'] 428 | photo['focal'] = r['focal'] 429 | photo['title'] = r['title'] 430 | res.append(photo) 431 | except Exception as e: 432 | logger.exception(e) 433 | res = None 434 | finally: 435 | db.close() 436 | return res 437 | 438 | def photo_exists_in_fs(self, photo): 439 | res = False 440 | try: 441 | 442 | # check if exists 1x in big 443 | big_path = os.path.join(self.cb.conf['lycheepath'], 'uploads', 'big', photo) 444 | assert os.path.exists(big_path), "Does not exists {}".format(big_path) 445 | # check if exists 2x in thumbnail 446 | thumb_path = os.path.join(self.cb.conf['lycheepath'], 'uploads', 'thumb', photo) 447 | assert os.path.exists(thumb_path), "Does not exists {}".format(thumb_path) 448 | file, ext = photo.split('.') 449 | thumb_path = os.path.join( 450 | self.cb.conf['lycheepath'], 'uploads', 'thumb', ''.join([file, '@2x.', ext])) 451 | assert os.path.exists(thumb_path), "Does not exists {}".format(thumb_path) 452 | res = True 453 | except Exception as e: 454 | logger.exception(e) 455 | finally: 456 | return res 457 | 458 | def photo_exists_in_db(self, photo_id): 459 | res = False 460 | try: 461 | if (self.get_photos(p_id=photo_id)): 462 | res = True 463 | 464 | except Exception as e: 465 | logger.exception(e) 466 | finally: 467 | return res 468 | 469 | def get_album_ids_titles(self): 470 | res = None 471 | db = self._connect_db() 472 | try: 473 | # check if exists in db 474 | sql = "select id, title from lychee_albums" 475 | with db.cursor() as cursor: 476 | cursor.execute(sql) 477 | rows = cursor.fetchall() 478 | res = rows 479 | except Exception as e: 480 | # logger.exception(e) 481 | res = None 482 | raise e 483 | finally: 484 | db.close() 485 | return res 486 | 487 | def check_album_size(self, a_name): 488 | res = None 489 | try: 490 | a_id = self.get_album_id(a_name) 491 | assert a_id, "Album does not exist in db" 492 | # check no of photo in db 493 | photos = self.get_photos(a_id) 494 | nb_photos_in_db = len(photos) 495 | # check if files exists on fs 496 | for p in photos: 497 | assert self.photo_exists_in_fs(p['url']), "All photos for {} are not on fs".format(photos) 498 | # every thing is ok 499 | res = nb_photos_in_db 500 | except Exception as e: 501 | logger.exception(e) 502 | finally: 503 | return res 504 | 505 | def get_column_width(self, table, column): 506 | res = 50 # default value 507 | query = "show columns from " + table + " where Field='" + column + "'" 508 | logger.info(query) 509 | db = self._connect_db() 510 | cur = db.cursor() 511 | try: 512 | cur.execute(query) 513 | row = cur.fetchone() 514 | logger.info(row) 515 | type = row['Type'] 516 | # is type ok 517 | p = re.compile('varchar\(\d+\)', re.IGNORECASE) 518 | if p.match(type): 519 | # remove varchar(and) 520 | p = re.compile('\d+', re.IGNORECASE) 521 | ints = p.findall(type) 522 | if len(ints) > 0: 523 | res = int(ints[0]) 524 | else: 525 | logger.ERROR( 526 | "unable to find column width for " + 527 | table + 528 | "." + 529 | column + 530 | " fallback to default") 531 | except Exception as e: 532 | logger.exception(e) 533 | logger.error("Impossible to find column width for " + table + "." + column) 534 | finally: 535 | db.close() 536 | return res 537 | 538 | def change_column_width(self, table, column, width): 539 | 540 | if width: 541 | 542 | try: 543 | query = "alter " + table + " modify " + column + " varchar(" + str(width) + ")" 544 | db = self._connect_db() 545 | self._exec_sql(db, query) 546 | except Exception as e: 547 | logger.exception(e) 548 | logger.error( 549 | "Impossible to modify column width for " + 550 | table + 551 | "." + 552 | column + 553 | " to " + 554 | str(width)) 555 | raise 556 | finally: 557 | db.close() 558 | --------------------------------------------------------------------------------