├── .editorconfig ├── .gitignore ├── LICENSE ├── README.md ├── final_result.png └── src ├── __init__.py ├── config.json.example ├── organize_media_files.py ├── requirements.txt └── utils ├── __init__.py ├── generic.py └── logging.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://editorconfig.org/ 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | end_of_line = lf 11 | charset = utf-8 12 | 13 | # Docstrings and comments use max_line_length = 79 14 | [*.py] 15 | max_line_length = 119 16 | 17 | # Use 2 spaces for the HTML files 18 | [*.html] 19 | indent_size = 2 20 | 21 | # The JSON files contain newlines inconsistently 22 | [*.json] 23 | indent_size = 2 24 | insert_final_newline = ignore 25 | 26 | [**/admin/js/vendor/**] 27 | indent_style = ignore 28 | indent_size = ignore 29 | 30 | # Minified JavaScript files shouldn't be changed 31 | [**.min.js] 32 | indent_style = ignore 33 | insert_final_newline = ignore 34 | 35 | # Makefiles always use tabs for indentation 36 | [Makefile] 37 | indent_style = tab 38 | 39 | # Batch files use tabs for indentation 40 | [*.bat] 41 | indent_style = tab 42 | 43 | [docs/**.txt] 44 | max_line_length = 79 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | 3 | *~ 4 | .DS_Store 5 | bin/* 6 | bin 7 | .svn 8 | gen 9 | gen/* 10 | Thumbs.db 11 | *.pyc 12 | *.diff 13 | *. 14 | *.class 15 | web/WEB-INF/classes 16 | .settings 17 | .idea 18 | .vagrant 19 | *.log 20 | /venv/ 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Valentino Pistis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OrganizeMediaFiles Alpha 2 | 3 | This project is a collection of Python scripts that help to organize media 4 | files into a directory tree "year/month" based on file metadata, using [exiftool](https://exiftool.org/) 5 | 6 | Used by me into my personal [Nextcloud](https://www.nextcloud.com) installation to organize unsorted files. 7 | Strongly inspired by: https://github.com/OneLogicalMyth/Random-Scripts/blob/master/NextCloud/SortPictures.py 8 | 9 | **Tested with Linux/Debian and MacOS python 2 and 3** 10 | 11 | **ATTENTION alpha version backup your files before use!** 12 | 13 | This picture describe the final result: 14 | ![final result](final_result.png) 15 | 16 | ## Features 17 | 18 | + manages duplicate files due to milliseconds difference (the new file uses the same old name). 19 | + support all file types supported by [exiftool](https://exiftool.org/) 20 | + proccess video and photo at same time (see `config.json`) 21 | + if exists, add sub second time original (fraction of seconds) to file name 22 | 23 | ## Getting OrganizeMediaFiles: 24 | 25 | Clone the repository: `git clone https://github.com/vpistis/OrganizeMediaFiles.git`. 26 | Alternatively download [tarball](https://github.com/vpistis/OrganizeMediaFiles/tarball/master) 27 | or [zip](https://github.com/vpistis/OrganizeMediaFiles/archive/master.zip). There haven't been any releases yet. 28 | 29 | ## Installation 30 | 31 | No installation required, it's a simple python script :) It run in `python`. 32 | 33 | ## Requirements 34 | 35 | In order to use the script, is necessary install at least [exiftool](https://exiftool.org/). 36 | Also install all `requirements.txt` 37 | 38 | ### Debian based 39 | 40 | `sudo apt-get install exiftool` 41 | 42 | ### MacOS 43 | 44 | `brew install exiftool` 45 | 46 | ## Usage 47 | 48 | ### Step 1: configuration 49 | 50 | Configure using `config.json` 51 | 52 | ### Step 2: run 53 | 54 | `python organize_media_files.py` 55 | 56 | ## Configuration 57 | 58 | **Important**: process only photo/video files with specified extensions. 59 | Use the `config.json` to change paths and other stuff. 60 | 61 | **All paths are intended without any escaping char,like backslashes** 62 | 63 | ### LOG_LEVEL 64 | 65 | Default: `"LOG_LEVEL": "INFO"` 66 | Log files are in the source directory 67 | 68 | ### LOG_PATH 69 | 70 | Default: `"LOG_DIR": "/tmp"`. 71 | If not specified use this directory. If you cannot write into this path, maybe the best choice is to use `/tmp` 72 | 73 | ### RENAME_SORTED_FILES 74 | 75 | Default: `"RENAME_SORTED_FILES": false`. 76 | If `true` rename sorted files according to metadata creation time. 77 | See `DATE_FORMAT_OUTPUT` for the name of new file. 78 | 79 | ### REMOVE_OLD_FILES 80 | 81 | Default: `"REMOVE_OLD_FILES": false`. 82 | If `true` delete old file after creation of new processed file. 83 | 84 | ### APPEND_ORIG_FILENAME 85 | 86 | Default: `"APPEND_ORIG_FILENAME": false`. 87 | If `true` append the original file name at the end of new created file. 88 | 89 | ### REMOVE_SPACE_FROM_FILENAME 90 | 91 | Default: `"REMOVE_SPACE_FROM_FILENAME": true`. 92 | Sometimes some space remains into filename, i want to replace it with underscores: `_` 93 | 94 | ### DATE_FORMAT_OUTPUT 95 | 96 | Default: `"DATE_FORMAT_OUTPUT": "%Y%m%d_%H%M%S"`. 97 | The date output string in python used to name the new created file. 98 | Used only if `RENAME_SORTED_FILE=true`. 99 | 100 | ### PROCESS_IMAGES 101 | 102 | Default: `"PROCESS_IMAGES": true`. 103 | If `true` process image files according to the `IMAGE_FILES_EXTENSIONS`. 104 | 105 | ### IMAGE_FILENAME_SUFFIX 106 | 107 | Default: `"IMAGE_FILENAME_SUFFIX": "IMG_"`. 108 | Prepend this string to the new created file name. 109 | 110 | ### IMAGES_SOURCE_PATH 111 | 112 | Default: `"IMAGES_SOURCE_PATH": "/media/drivemount/user/files/FilesToSort"`. 113 | The path where original image files are stored. 114 | 115 | ### IMAGES_DESTINATION_PATH 116 | 117 | Default: `"IMAGES_DESTINATION_PATH": "/media/drivemount/user/files/FilesSorted"`. 118 | The root path where store the new organized image files. 119 | 120 | ### IMAGE_FILES_EXTENSIONS 121 | 122 | Default: `"IMAGE_FILES_EXTENSIONS": [".jpg",".gif",".tiff"]`. 123 | Files with these extensions are processed as images (NOTE: exiftool support internally many file types, and the command 124 | used in this script is the same for images/videos) 125 | 126 | ### PROCESS_VIDEOS 127 | 128 | Default: `"PROCESS_VIDEOS": false`. 129 | If `true` process video files according to the `VIDEO_FILES_EXTENSIONS`. 130 | 131 | ### VIDEO_FILENAME_SUFFIX 132 | 133 | Default: `"VIDEO_FILENAME_SUFFIX": "VID_"`. 134 | Prepend this string to the new created file name. 135 | 136 | ### VIDEOS_SOURCE_PATH 137 | 138 | Default: `"VIDEOS_SOURCE_PATH": "/media/drivemount/user/files/FilesToSort"` 139 | The path where original video files are stored. 140 | 141 | ### VIDEOS_DESTINATION_PATH 142 | 143 | Default: `"VIDEOS_DESTINATION_PATH": "/media/drivemount/user/files/FilesSorted"`. 144 | The root path where store the new organized video files. 145 | 146 | ### VIDEO_FILES_EXTENSIONS 147 | 148 | Default: `"VIDEO_FILES_EXTENSIONS": [".mp4",".3gp"]`. 149 | Files with these extensions are processed as videos see [IMAGE_FILES_EXTENSIONS](#image_files_extensions) note. 150 | 151 | ### NEXTCLOUD 152 | 153 | Default: `"NEXTCLOUD": false`. 154 | If `true` exec nextcloud command to rescan data directory for new files. 155 | 156 | ### NEXTCLOUD_PATH 157 | 158 | Default: `"NEXTCLOUD_PATH": "/var/www/html/nextcloud"`. 159 | The path of nextcloud installation. 160 | 161 | ### NEXTCLOUD_USER 162 | 163 | Default: `"NEXTCLOUD_USER": "www-data"` 164 | Launch nextcloud scan command as a `"NEXTCLOUD_USER"`. 165 | 166 | # LICENSE 167 | 168 | MIT License 169 | 170 | Copyright (c) 2017-2023 Valentino Pistis 171 | 172 | Permission is hereby granted, free of charge, to any person obtaining a copy 173 | of this software and associated documentation files (the "Software"), to deal 174 | in the Software without restriction, including without limitation the rights 175 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 176 | copies of the Software, and to permit persons to whom the Software is 177 | furnished to do so, subject to the following conditions: 178 | 179 | The above copyright notice and this permission notice shall be included in all 180 | copies or substantial portions of the Software. 181 | 182 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 183 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 184 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 185 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 186 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 187 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 188 | SOFTWARE. 189 | -------------------------------------------------------------------------------- /final_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vpistis/OrganizeMediaFiles/4bb7699541245712cdba561a5568a66887a2cdcb/final_result.png -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Created on 24/01/17 10:24 5 | 6 | @author: vpistis 7 | """ 8 | -------------------------------------------------------------------------------- /src/config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "LOG_LEVEL": "INFO", 3 | "LOG_DIR": "/tmp", 4 | "RENAME_SORTED_FILES": false, 5 | "REMOVE_OLD_FILES": false, 6 | "APPEND_ORIG_FILENAME": false, 7 | "REMOVE_SPACE_FROM_FILENAME": true, 8 | "DATE_FORMAT_OUTPUT": "%Y%m%d_%H%M%S", 9 | "PROCESS_IMAGES": true, 10 | "IMAGE_FILENAME_SUFFIX": "IMG_", 11 | "IMAGES_SOURCE_PATH": "/media/drivemount/user/files/FilesToSort", 12 | "IMAGES_DESTINATION_PATH": "/media/drivemount/user/files/FilesSorted", 13 | "IMAGE_FILES_EXTENSIONS": [ 14 | ".jpg", 15 | ".gif", 16 | ".tiff" 17 | ], 18 | "PROCESS_VIDEOS": false, 19 | "VIDEO_FILENAME_SUFFIX": "VID_", 20 | "VIDEOS_SOURCE_PATH": "/media/drivemount/user/files/FilesToSort", 21 | "VIDEOS_DESTINATION_PATH": "/media/drivemount/user/files/FilesSorted", 22 | "VIDEO_FILES_EXTENSIONS": [ 23 | ".mp4", 24 | ".3gp", 25 | ".mov" 26 | ], 27 | "NEXTCLOUD": false, 28 | "NEXTCLOUD_PATH": "/var/www/html/nextcloud", 29 | "NEXTCLOUD_USER": "www-data", 30 | "NEXTCLOUD_GROUP": "www-data", 31 | "NC_CONFIG_PREVIEW_MAX_X": "1080", 32 | "NC_CONFIG_PREVIEW_MAX_Y": "1920", 33 | "NC_CONFIG_JPEG_QUALITY": "60", 34 | "NC_PREVIEW_SQUARESIZES": "32 256", 35 | "NC_PREVIEW_WIDTHSIZES": "256 384", 36 | "NC_PREVIEW_HEIGHTSIZES": "256", 37 | "NEXTCLOUD_USER": "www-data" 38 | } 39 | -------------------------------------------------------------------------------- /src/organize_media_files.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Organize pictures file into directory tree with year and month. 5 | Use perl exiftool to get creation date and filename from file metadata. 6 | 7 | Strongly inspired from the project: 8 | https://github.com/OneLogicalMyth/Random-Scripts.git 9 | 10 | Created on 27/12/16 15:53 11 | 12 | @author: vpistis 13 | """ 14 | 15 | import datetime 16 | import filecmp 17 | import os 18 | import shutil 19 | import subprocess 20 | import timeit 21 | from utils.generic import get_setting, which 22 | from utils import logging 23 | 24 | # from utils import get_access_token 25 | logger = logging.get_logger("main") 26 | log_file = logging.get_logger_file("main") 27 | # sys.stdout = Logger() 28 | 29 | PROCESS_IMAGES = get_setting("PROCESS_IMAGES") 30 | PROCESS_VIDEOS = get_setting("PROCESS_VIDEOS") 31 | 32 | IMAGES_SOURCE_PATH = get_setting("IMAGES_SOURCE_PATH") 33 | IMAGES_DESTINATION_PATH = get_setting("IMAGES_DESTINATION_PATH") 34 | IMAGE_FILES_EXTENSIONS = tuple(get_setting("IMAGE_FILES_EXTENSIONS")) 35 | IMAGE_FILENAME_SUFFIX = get_setting("IMAGE_FILENAME_SUFFIX") 36 | 37 | VIDEOS_SOURCE_PATH = get_setting("VIDEOS_SOURCE_PATH") 38 | VIDEOS_DESTINATION_PATH = get_setting("VIDEOS_DESTINATION_PATH") 39 | VIDEO_FILES_EXTENSIONS = tuple(get_setting("VIDEO_FILES_EXTENSIONS")) 40 | VIDEO_FILENAME_SUFFIX = get_setting("VIDEO_FILENAME_SUFFIX") 41 | 42 | # If false copy file and don't remove old file 43 | REMOVE_OLD_FILES = get_setting("REMOVE_OLD_FILES") 44 | APPEND_ORIG_FILENAME = get_setting("APPEND_ORIG_FILENAME") 45 | REMOVE_SPACE_FROM_FILENAME = get_setting("REMOVE_SPACE_FROM_FILENAME") 46 | # if RENAME_SORTED_FILES=False, use this date format for naming files 47 | DATE_FORMAT_OUTPUT = get_setting("DATE_FORMAT_OUTPUT") 48 | # if false, sorted files keep their original name, else rename using CreateDate 49 | RENAME_SORTED_FILES = get_setting("RENAME_SORTED_FILES") 50 | # in case you use nextcloud or owncloud, set NEXTCLOUD=True to rescan all files 51 | NEXTCLOUD = get_setting("NEXTCLOUD") 52 | NEXTCLOUD_PATH = get_setting("NEXTCLOUD_PATH") 53 | NEXTCLOUD_USER = get_setting("NEXTCLOUD_USER") 54 | NEXTCLOUD_GROUP = get_setting("NEXTCLOUD_GROUP") 55 | 56 | # in case you selected NEXTCLOUD=True, this will generate thumbnails 57 | NC_CONFIG_PREVIEW_MAX_X = get_setting("NC_CONFIG_PREVIEW_MAX_X") 58 | NC_CONFIG_PREVIEW_MAX_Y = get_setting("NC_CONFIG_PREVIEW_MAX_Y") 59 | NC_CONFIG_JPEG_QUALITY = get_setting("NC_CONFIG_JPEG_QUALITY") 60 | NC_PREVIEW_SQUARESIZES = get_setting("NC_PREVIEW_SQUARESIZES") 61 | NC_PREVIEW_WIDTHSIZES = get_setting("NC_PREVIEW_WIDTHSIZES") 62 | NC_PREVIEW_HEIGHTSIZES = get_setting("NC_PREVIEW_HEIGHTSIZES") 63 | 64 | 65 | def get_create_date(filename): 66 | """ 67 | Get creation date from file metadata 68 | 69 | :param filename: 70 | :return: [day, month, year, "%Y:%m:%d %H:%M:%S"] 71 | """ 72 | command = ["exiftool", "-CreateDate", "-s3", "-fast2", filename] 73 | create_date = subprocess.check_output(command, universal_newlines=True) 74 | metadata = create_date 75 | logger.debug("command: {}".format(command)) 76 | logger.debug("create_date: {}".format(create_date)) 77 | datetime_original = None 78 | metadata = None 79 | 80 | if not create_date: 81 | command = ["exiftool", "-DateTimeOriginal", "-s3", "-fast2", filename] 82 | datetime_original = subprocess.check_output(command, universal_newlines=True) 83 | metadata = datetime_original 84 | logger.debug("command: {}".format(command)) 85 | logger.debug("datetime_original: {}".format(datetime_original)) 86 | if not metadata and not datetime_original: 87 | command = ["exiftool", "-filemodifydate", "-s3", "-fast2", filename] 88 | file_modify_date = subprocess.check_output(command, universal_newlines=True) 89 | metadata = file_modify_date.split("+")[0] 90 | logger.debug("command: {}".format(command)) 91 | logger.debug("file_modify_date: {}".format(file_modify_date)) 92 | 93 | try: 94 | # Grab date taken 95 | datetaken_object = datetime.datetime.strptime(metadata.rstrip(), "%Y:%m:%d %H:%M:%S") 96 | 97 | # Date 98 | day = str(datetaken_object.day).zfill(2) 99 | month = str(datetaken_object.month).zfill(2) 100 | year = str(datetaken_object.year) 101 | 102 | # New Filename 103 | output = [day, month, year, datetaken_object.strftime(DATE_FORMAT_OUTPUT)] 104 | logger.debug("output: {}".format(output)) 105 | return output 106 | 107 | except Exception as e: 108 | logger.exception("{}".format(e)) 109 | logger.exception("exiftool is not installed or datetime is unknown") 110 | return None 111 | 112 | 113 | def get_sub_sec_time_original(filename): 114 | """ 115 | Get SubSecTimeOriginal from file metadata if exists 116 | 117 | :param filename: 118 | :return: 119 | """ 120 | try: 121 | command = ["exiftool", "-SubSecTimeOriginal", "-s3", "-fast2", filename] 122 | metadata = subprocess.check_output(command, universal_newlines=True) 123 | logger.debug("command: {}".format(command)) 124 | logger.debug("metadata: {}".format(metadata)) 125 | return metadata.rstrip() 126 | except Exception as e: 127 | logger.exception("{}".format(e)) 128 | logger.exception("exiftool is installed?") 129 | return None 130 | 131 | 132 | def get_file_name(filename): 133 | """ 134 | Get real filename from metadata 135 | 136 | :param filename: 137 | :return: 138 | """ 139 | try: 140 | command = ["exiftool", "-filename", "-s3", "-fast2", filename] 141 | metadata = subprocess.check_output(command, universal_newlines=True) 142 | logger.debug("command: {}".format(command)) 143 | logger.debug("metadata: {}".format(metadata)) 144 | return metadata.rstrip() 145 | except Exception as e: 146 | logger.exception("{}".format(e)) 147 | logger.exception("exiftool is installed?") 148 | return None 149 | 150 | 151 | def get_file_ext(filename): 152 | """ 153 | Return the file extension based on file name from metadata, include point. 154 | Example return: '.jpg' 155 | 156 | :param filename: 157 | :return: 158 | """ 159 | extension = ".{}".format(get_file_name(filename).split(".")[-1]) 160 | logger.debug("file_extension: {}".format(extension)) 161 | return extension 162 | 163 | 164 | def organize_files(src_path, dest_path, files_extensions, filename_suffix=""): 165 | """ 166 | Get all files from directory and process 167 | 168 | :return: 169 | """ 170 | _src_path = src_path 171 | _dest_path = dest_path 172 | _files_extensions = files_extensions 173 | _filename_suffix = filename_suffix 174 | 175 | # check if destination path is existing create if not 176 | if not os.path.exists(_dest_path): 177 | os.makedirs(_dest_path) 178 | logger.debug("Destination path created: {}".format(_dest_path)) 179 | 180 | if len(os.listdir(_src_path)) <= 0: 181 | logger.warning("No files in path: {}".format(_src_path)) 182 | return 0, 0, 0, 0 183 | else: 184 | num_files_processed = 0 185 | num_files_removed = 0 186 | num_files_copied = 0 187 | num_files_skipped = 0 188 | 189 | for file in os.listdir(_src_path): 190 | 191 | abs_file_path = "{}/{}".format(_src_path, file) 192 | logger.info("File found: {}".format(abs_file_path)) 193 | 194 | if os.path.isdir(abs_file_path): 195 | logger.info("Found a directory {} ...searching in it for new files.".format(abs_file_path)) 196 | _num_files_processed, _num_files_removed, _num_files_copied, _num_files_skipped = organize_files( 197 | abs_file_path, _dest_path, _files_extensions, _filename_suffix) 198 | 199 | num_files_processed += _num_files_processed 200 | num_files_removed += _num_files_removed 201 | num_files_copied += _num_files_copied 202 | num_files_skipped += _num_files_skipped 203 | 204 | elif os.path.isfile(abs_file_path) and file.lower().endswith(_files_extensions): 205 | 206 | num_files_processed += 1 207 | 208 | filename = _src_path + os.sep + file 209 | file_ext = get_file_ext(filename) 210 | date_info = get_create_date(filename) 211 | if not date_info: 212 | logger.warning("Skipped No Creation Date metadata for file: {}".format(abs_file_path)) 213 | continue 214 | try: 215 | out_filepath = _dest_path + os.sep + date_info[2] + os.sep + date_info[1] 216 | if RENAME_SORTED_FILES: 217 | if APPEND_ORIG_FILENAME: 218 | out_filename = out_filepath + os.sep + _filename_suffix + date_info[3] \ 219 | + get_sub_sec_time_original(filename) + '_' + file 220 | else: 221 | out_filename = out_filepath + os.sep + _filename_suffix + date_info[3] \ 222 | + get_sub_sec_time_original(filename) + file_ext 223 | else: 224 | if APPEND_ORIG_FILENAME: 225 | out_filename = out_filepath + os.sep + get_file_name(filename) + '_' + file 226 | else: 227 | out_filename = out_filepath + os.sep + get_file_name(filename) 228 | # sometimes into the original file name there is spaces, and i wanto to remove it 229 | if REMOVE_SPACE_FROM_FILENAME: 230 | out_filename = "{}{}".format(out_filename.split(file_ext)[0].strip(" ").replace(" ", "_"), file_ext) 231 | # check if destination path is existing create if not 232 | if not os.path.exists(out_filepath): 233 | os.makedirs(out_filepath) 234 | 235 | # don't overwrite files if the name is the same 236 | if os.path.exists(out_filename): 237 | shutil.copy2(filename, out_filename + '_duplicate') 238 | if filecmp.cmp(filename, out_filename + '_duplicate', shallow=False): 239 | # the old file name exists...skip file 240 | os.remove(out_filename + '_duplicate') 241 | num_files_skipped += 1 242 | logger.warning("Skipped file: {}".format(filename)) 243 | continue 244 | else: 245 | # new dest path but old filename, file duplicate i the destination 246 | out_filename = out_filepath + os.sep + file 247 | 248 | if os.path.exists(out_filename): 249 | shutil.copy2(filename, out_filename + '_duplicate') 250 | if filecmp.cmp(filename, out_filename + '_duplicate', shallow=False): 251 | # the old file name exists...skip file 252 | os.remove(out_filename + '_duplicate') 253 | num_files_skipped += 1 254 | logger.warning("Skipped file: {}".format(filename)) 255 | continue 256 | 257 | # copy the file to the organised structure 258 | shutil.copy2(filename, out_filename) 259 | if filecmp.cmp(filename, out_filename, shallow=False): 260 | num_files_copied += 1 261 | logger.info('File {} copied with success to {}'.format(num_files_copied, out_filename)) 262 | if REMOVE_OLD_FILES: 263 | os.remove(filename) 264 | num_files_removed += 1 265 | logger.info('Removed old file {}'.format(filename)) 266 | else: 267 | logger.warning('File failed to copy :( {}'.format(filename)) 268 | 269 | except Exception as e: 270 | logger.exception(e) 271 | logger.exception("Exception occurred") 272 | return num_files_processed, num_files_removed, num_files_copied, num_files_skipped 273 | except None: 274 | logger.exception('File has no metadata skipped {}'.format(filename)) 275 | 276 | return num_files_processed, num_files_removed, num_files_copied, num_files_skipped 277 | 278 | 279 | # Nextcloud initiate a scan 280 | def nextcloud_files_scan(): 281 | logger.info("Scan Nextcloud files...") 282 | try: 283 | subprocess.Popen("sudo -u {} php {}/console.php files:scan --all".format(NEXTCLOUD_USER, NEXTCLOUD_PATH), 284 | shell=True, stdout=subprocess.PIPE) 285 | 286 | #then regenerate thumbnail (trully recomended on small servers) 287 | #source 1 https://www.bentasker.co.uk/posts/documentation/linux/671-improving-nextcloud-s-thumbnail-response-time.html 288 | #source 2 https://rayagainstthemachine.net/linux%20administration/nextcloud-photos/ 289 | subprocess.Popen("cd {}/apps;\ 290 | chown -R {}:{} ./previewgenerator \ 291 | sudo -u {} bash \ 292 | git clone https://github.com/rullzer/previewgenerator.git; \ 293 | cd ..; \ 294 | php --define apc.enable_cli=1 ./occ config:system:set preview_max_x --value {}; \ 295 | php --define apc.enable_cli=1 ./occ config:system:set preview_max_y --value {};\ 296 | php --define apc.enable_cli=1 ./occ config:system:set jpeg_quality --value {};\ 297 | php --define apc.enable_cli=1 ./occ config:app:set --value="{}" previewgenerator squareSizes; \ 298 | php --define apc.enable_cli=1 ./occ config:app:set --value="{}" previewgenerator widthSizes; \ 299 | php --define apc.enable_cli=1 ./occ config:app:set --value="{}" previewgenerator heightSizes; \ 300 | php --define apc.enable_cli=1 ./occ preview:generate-all -vvv; 301 | ".format( NEXTCLOUD_PATH, \ 302 | NEXTCLOUD_USER, NEXTCLOUD_GROUP, \ 303 | NEXTCLOUD_USER, \ 304 | NC_CONFIG_PREVIEW_MAX_X, \ 305 | NC_CONFIG_PREVIEW_MAX_Y, \ 306 | NC_CONFIG_JPEG_QUALITY, \ 307 | NC_PREVIEW_SQUARESIZES, \ 308 | NC_PREVIEW_WIDTHSIZES, \ 309 | NC_PREVIEW_HEIGHTSIZES), 310 | shell=True, stdout=subprocess.PIPE) 311 | except Exception as e: 312 | logger.exception(e) 313 | logger.exception("Exception occurred") 314 | return 315 | 316 | 317 | def main(): 318 | # check if exiftool is installed 319 | if not which("exiftool"): 320 | logger.error("Please...install exiftool first") 321 | return 322 | 323 | logger.info("======== {} =======".format(datetime.datetime.now())) 324 | if PROCESS_IMAGES: 325 | logger.info("Start process images...") 326 | start_time = timeit.default_timer() 327 | img_processed, img_removed, img_copied, img_skipped = organize_files(IMAGES_SOURCE_PATH, 328 | IMAGES_DESTINATION_PATH, 329 | IMAGE_FILES_EXTENSIONS, 330 | IMAGE_FILENAME_SUFFIX) 331 | img_elapsed = timeit.default_timer() - start_time 332 | 333 | if PROCESS_VIDEOS: 334 | logger.info("Start process videos...") 335 | start_time = timeit.default_timer() 336 | vid_processed, vid_removed, vid_copied, vid_skipped = organize_files(VIDEOS_SOURCE_PATH, 337 | VIDEOS_DESTINATION_PATH, 338 | VIDEO_FILES_EXTENSIONS, 339 | VIDEO_FILENAME_SUFFIX) 340 | vid_elapsed = timeit.default_timer() - start_time 341 | if PROCESS_IMAGES: 342 | logger.info("End process IMAGES in: {} seconds.".format(img_elapsed)) 343 | logger.info("Processed: {}. Removed: {}. Copied: {}. Skipped: {}".format(img_processed, 344 | img_removed, img_copied, img_skipped)) 345 | logger.info("Image files are stored in: {}".format(IMAGES_DESTINATION_PATH)) 346 | if PROCESS_VIDEOS: 347 | logger.info("End process VIDEOS in: {} seconds.".format(vid_elapsed)) 348 | logger.info("Processed: {}. Removed: {}. Copied: {}. Skipped: {}".format(vid_processed, 349 | vid_removed, vid_copied, vid_skipped)) 350 | logger.info("Video files are stored in: {}".format(VIDEOS_DESTINATION_PATH)) 351 | if NEXTCLOUD: 352 | nextcloud_files_scan() 353 | 354 | return 355 | 356 | 357 | # Execution 358 | main() 359 | -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | pathlib==1.0.1 2 | -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vpistis/OrganizeMediaFiles/4bb7699541245712cdba561a5568a66887a2cdcb/src/utils/__init__.py -------------------------------------------------------------------------------- /src/utils/generic.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Created on 30/12/16 21:42 5 | 6 | @author: vpistis 7 | """ 8 | import json 9 | import os 10 | 11 | import sys 12 | 13 | 14 | def get_setting(key): 15 | """Get the secret variable or return explicit exception.""" 16 | try: 17 | base_dir = str(os.path.dirname(__file__)) 18 | print("BASE DIR: {}".format(base_dir)) 19 | print("KEY: {}".format(key)) 20 | print("config.json: {}".format(os.path.join(base_dir, "../config.json"))) 21 | with open(os.path.join(base_dir, "../config.json")) as f: 22 | config_json = json.loads(f.read()) 23 | 24 | return config_json[key] 25 | except KeyError: 26 | error_msg = "Set the {0} environment variable".format(key) 27 | raise KeyError(error_msg) 28 | except Exception as e: 29 | raise Exception("Some error occurred: ", e) 30 | 31 | 32 | def which(program): 33 | """ 34 | Check if a program/executable exists 35 | 36 | :param program: 37 | :return: 38 | """ 39 | 40 | def is_exe(f_path): 41 | return os.path.isfile(f_path) and os.access(f_path, os.X_OK) 42 | 43 | fpath, fname = os.path.split(program) 44 | 45 | if fpath: 46 | if is_exe(program): 47 | return program 48 | else: 49 | for path in os.environ["PATH"].split(os.pathsep): 50 | path = path.strip('"') 51 | exe_file = os.path.join(path, program) 52 | if is_exe(exe_file): 53 | return exe_file 54 | 55 | return None 56 | 57 | 58 | class Logger(object): 59 | """ 60 | http://stackoverflow.com/a/14906787/5941790 61 | """ 62 | 63 | def __init__(self): 64 | self.terminal = sys.stdout 65 | log_file_path = get_setting("LOG_FILE") 66 | print("LOG_FILE PATH: {}".format(log_file_path)) 67 | self.log = open(log_file_path, "a") 68 | 69 | def write(self, message): 70 | self.terminal.write(message) 71 | self.log.write(message) 72 | 73 | def flush(self): 74 | # this flush method is needed for python 3 compatibility. 75 | # this handles the flush command by doing nothing. 76 | # you might want to specify some extra behavior here. 77 | pass 78 | -------------------------------------------------------------------------------- /src/utils/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import date, datetime 4 | from pathlib import WindowsPath 5 | 6 | """ 7 | Internal Module Logger 8 | """ 9 | from utils.generic import get_setting 10 | try: 11 | LOG_DIR = os.path.join(get_setting("LOG_DIR"), "logs", date.today().strftime("%Y-%m-%d")) 12 | except: 13 | BASEPATH = os.path.dirname(__file__) 14 | print("BASEPATH: {}".format(BASEPATH)) 15 | LOG_DIR = os.path.join(os.path.dirname(BASEPATH), "logs", date.today().strftime("%Y-%m-%d")) 16 | 17 | POSIX = 'posix' 18 | # LOG_LEVEL_INFO = logging.INFO 19 | # LOG_LEVEL_DEBUG = logging.DEBUG 20 | LOGGER_DATEFORMAT = "%Y:%m:%d %H:%M:%S" 21 | try: 22 | LOG_LEVEL = get_setting("LOG_LEVEL") 23 | except: 24 | LOG_LEVEL = "DEBUG" 25 | 26 | 27 | class MainLogger(object): 28 | """ 29 | holds the logger reference 30 | """ 31 | logger = None 32 | file = None 33 | 34 | 35 | def check_dir_exists(dir_path, raise_exception=False): 36 | """ 37 | Checks if a directory exists; otherwise it tries to create the directory 38 | :param dir_path: 39 | :param raise_exception: 40 | :return: 41 | """ 42 | try: 43 | if not os.path.exists(dir_path): 44 | os.makedirs(dir_path) 45 | print("dir {} CREATED!".format(dir_path)) 46 | else: 47 | print("Dir {} EXISTS!".format(dir_path)) 48 | except Exception as e: 49 | print("dir not created: {}".format(dir_path)) 50 | print("Exception: {}".format(e)) 51 | if raise_exception: 52 | raise e 53 | 54 | 55 | def get_logger_file(log_name): 56 | check_dir_exists(LOG_DIR) 57 | get_logger(log_name) 58 | return MainLogger.file 59 | 60 | 61 | def get_logger(log_name): 62 | """ 63 | Return logger reference 64 | :return: 65 | """ 66 | try: 67 | check_dir_exists(LOG_DIR) 68 | print("Logging logger: {}".format(logging.getLogger(log_name))) 69 | # if logging.getLogger(log_name) is None: 70 | if MainLogger.logger is None or MainLogger.logger != logging.getLogger(log_name): 71 | print("Regenerate Logger") 72 | logger = logging.getLogger(log_name) 73 | logger.setLevel(LOG_LEVEL) 74 | 75 | if os.name == POSIX: 76 | print("LOG_DIR: {}".format(LOG_DIR)) 77 | log_file = "{}/{}_{}.log".format(LOG_DIR, log_name, 78 | datetime.now()).replace(" ", "_").replace(":", "") 79 | print("LOG FILE: {}".format(log_file)) 80 | log_file_path = log_file 81 | else: 82 | print("LOG_DIR: {}".format(LOG_DIR)) 83 | log_file = "{}_{}.log".format(log_name, 84 | datetime.now()).replace(" ", "_").replace(":", "") 85 | print("LOG FILE: {}".format(log_file)) 86 | log_file_path = WindowsPath(os.path.join(LOG_DIR, log_file)) 87 | 88 | file = logging.FileHandler(log_file_path) 89 | file.setLevel(LOG_LEVEL) 90 | fileformat = logging.Formatter( 91 | "%(asctime)s [%(levelname)s] - [%(filename)s > %(funcName)s() > %(lineno)s] - %(message)s", 92 | datefmt=LOGGER_DATEFORMAT) 93 | file.setFormatter(fileformat) 94 | logger.addHandler(file) 95 | MainLogger.logger = logger 96 | MainLogger.file = log_file 97 | print("Main logger: {}".format(MainLogger.logger)) 98 | return MainLogger.logger 99 | except Exception as e: 100 | print("Exception occurred: {}".format(e)) 101 | return None 102 | --------------------------------------------------------------------------------