├── LICENSE ├── README.md └── flickr-backup.sh /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Roberto Aloi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flickr-backup 2 | 3 | A Bash script built on top of the flickcurl[1] library to ease backup 4 | of entire directories of photos as Flickr[2] photosets. 5 | 6 | To prevent duplicates, each photo is tagged with its own SHA1 during 7 | upload and a check is performed before every upload to avoid storing 8 | the same photo multiple times on Flickr. This allows the user to save 9 | bandwidth, storage and time. 10 | 11 | This simple approach makes interrupting and resuming an upload job 12 | a cheap operation and it provides what can be defined as a basic 'sync' 13 | operation between a directory on the file system and a photoset on 14 | Flickr. 15 | 16 | All operations performed by `flickr-backup` are logged to disk for 17 | reference. 18 | 19 | ## Usage 20 | 21 | ```` 22 | ./flickr-backup.sh DIRECTORY ALBUM 23 | ```` 24 | 25 | Upload all photos contained in DIRECTORY to the ALBUM photoset on 26 | Flickr. A new photoset is created every time the command is invoked, 27 | but only photos which are not already available on Flickr are 28 | uploaded, whilst the existing ones are simply added to the new photoset. 29 | 30 | The script outputs a `.` (dot) for each uploaded photo and a `x` for 31 | each skipped photo (when a duplicate is detected). 32 | 33 | Detailed logs are available for each upload job. The default logs 34 | location is set to `~/.flickrbackup`. 35 | 36 | ## Dependencies 37 | 38 | The script requires the flickcurl[1] library. On Mac, you can install 39 | it via _homebrew_[3]: 40 | 41 | ```` 42 | brew install flickcurl 43 | ```` 44 | 45 | ## References 46 | 47 | [1] http://librdf.org/flickcurl/ 48 | [2] http://flickr.com/ 49 | [3] http://brew.sh/ 50 | -------------------------------------------------------------------------------- /flickr-backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################################################ 4 | # Config 5 | ################################################################################ 6 | 7 | # Directory where logs will be stored. 8 | # It will be created if it does not exist. 9 | logdir="$HOME/.flickrbackup" 10 | 11 | ################################################################################ 12 | # Usage 13 | ################################################################################ 14 | 15 | function usage { 16 | echo -e "Usage: $0 DIRECTORY ALBUM" 17 | } 18 | 19 | ################################################################################ 20 | # Global Variables 21 | ################################################################################ 22 | 23 | PHOTO_ID= 24 | PHOTOSET_ID= 25 | TO_ADD= 26 | 27 | ################################################################################ 28 | # Counters 29 | ################################################################################ 30 | 31 | PHOTOS_SKIPPED=0 32 | PHOTOS_UNSUPPORTED=0 33 | PHOTOS_UPLOADED=0 34 | PHOTOS_ERROR=0 35 | PHOTOS_DUPLICATED=0 36 | 37 | ################################################################################ 38 | # Flickcurl Wrappers 39 | ################################################################################ 40 | 41 | # Given a name, creates a new photoset, using PHOTO_ID as a primary photo id. 42 | # Sets PHOTOSET_ID with the id of the created photoset. 43 | function photoset_create { 44 | 45 | # Handle function arguments 46 | photoset_name=$1 47 | 48 | # Hard-coded description for the photoset for now 49 | desc="This set is automatically generated" 50 | 51 | # Create a photoset from a primary photo 52 | res=$(flickcurl -q photosets.create "$photoset_name" "$desc" "$PHOTO_ID") 53 | 54 | # Extract and return PHOTOSET_ID 55 | PHOTOSET_ID=$(echo "$res" | cut -d' ' -f 3) 56 | 57 | # Log event 58 | log "Create photoset: $PHOTOSET_ID from primary: $PHOTO_ID" 59 | 60 | } 61 | 62 | # Adds PHOTO_ID to PHOTOSET_ID 63 | function photoset_add { 64 | 65 | # Add photo to photoset 66 | flickcurl -q photosets.addPhoto "$PHOTOSET_ID" "$PHOTO_ID" 67 | if [ $? == 0 ]; then 68 | # Log event 69 | log "Add photo: $PHOTO_ID to photoset: $PHOTOSET_ID" 70 | else 71 | log "Error adding photo: $PHOTO_ID to photoset: $PHOTOSET_ID" 72 | # The Flickr API is flaky. Retry one more time. 73 | flickcurl -q photosets.addPhoto "$PHOTOSET_ID" "$PHOTO_ID" 74 | if [ $? == 0 ]; then 75 | # Log event 76 | log "[Retry #1] Add photo: $PHOTO_ID to photoset: $PHOTOSET_ID" 77 | else 78 | log "[Retry #1] Error adding photo: $PHOTO_ID to photoset: $PHOTOSET_ID" 79 | log "[Retry #1] Giving up." 80 | fi 81 | fi 82 | 83 | } 84 | 85 | # Echoes how many photos are contained in PHOTOSET_ID 86 | function photoset_size { 87 | 88 | # Retrieve list of photos from photoset 89 | res=$(flickcurl photosets.getPhotos "$PHOTOSET_ID") 90 | 91 | # Extract and return total number of photos from result 92 | n=$(echo "$res"| head -2 | tail -1 | cut -d' ' -f 8 | rev | cut -c 2- | rev) 93 | echo "$n" 94 | 95 | } 96 | 97 | # Looks for a photo by SHA1. If a match exists, sets PHOTO_ID to the id of the 98 | # matching photo. If no match is found, unsets PHOTO_ID. 99 | function photo_lookup { 100 | 101 | # Handle function arguments 102 | tags=$1 103 | 104 | # Only search among user's own photos 105 | user="me" 106 | 107 | # Search photos by tags and check whether at least one entry is found 108 | res=$(flickcurl -q photos.search user "$user" tags "$tags") 109 | found=$(echo "$res" | grep -m 1 ID) 110 | 111 | # If a match is found, return the matching id 112 | if [ "$?" == "0" ]; then 113 | PHOTO_ID=$(echo "$found" | cut -d' ' -f6) 114 | log "Found matching tags: $tags (photo: $PHOTO_ID)" 115 | else 116 | PHOTO_ID= 117 | fi 118 | 119 | } 120 | 121 | # Uploads a photo and sets PHOTO_ID to the id of the created photo 122 | function photo_upload { 123 | 124 | # Handle function arguments 125 | file=$1 126 | tags=$2 127 | 128 | # Upload photo to Flickr 129 | res=$(flickcurl -q upload "$file" hidden hidden tags "$tags" 2>&1) 130 | 131 | if [ "$?" == "0" ]; then 132 | PHOTO_ID=$(echo $res | rev | cut -d' ' -f1 | rev) 133 | log "Upload photo: $PHOTO_ID with tags: $tags" 134 | else 135 | PHOTO_ID= 136 | log "Upload error: $res" 137 | fi 138 | 139 | } 140 | 141 | ################################################################################ 142 | # Helpers 143 | ################################################################################ 144 | 145 | function line { 146 | echo "================================================================" 147 | } 148 | 149 | function progress { 150 | echo -n "." 151 | } 152 | 153 | function skip { 154 | echo -n "x" 155 | } 156 | 157 | function unsupported { 158 | echo -n "U" 159 | } 160 | 161 | function duplicate { 162 | echo -n "D" 163 | } 164 | 165 | function error { 166 | echo -n "E" 167 | } 168 | 169 | function timestamp { 170 | echo -n "$(date +%s)" 171 | } 172 | 173 | function datetime { 174 | echo -n "$(date +%F-%T)" 175 | } 176 | 177 | function result { 178 | 179 | msg=$1 180 | result=$2 181 | 182 | log "$(line)" 183 | log "$msg" 184 | log "$(line)" 185 | 186 | tail -n 10 "$logfile" 187 | echo "Complete logs are available at: $logfile" 188 | 189 | exit "$result" 190 | 191 | } 192 | 193 | function sha1 { 194 | 195 | # Handle function arguments 196 | filename=$1 197 | 198 | sha=$(sha1sum "$filename" | cut -d' ' -f1) 199 | log "SHA1 for: $filename is: $sha" 200 | echo "$sha" 201 | } 202 | 203 | function require_program { 204 | 205 | # Handle function arguments 206 | program=$1 207 | url=$2 208 | 209 | # Exit if a required program is missing 210 | command -v "$program" > /dev/null 2>&1 || { 211 | echo "Program $program is required. More info at: $url" >&2 212 | exit 1 213 | } 214 | 215 | } 216 | 217 | function photo_upload_nodup { 218 | 219 | # Handle function arguments 220 | file=$1 221 | 222 | extension="${file##*.}" 223 | if [ "$extension" == "HEIC" ]; then 224 | heic_to_jpg "$file"; 225 | file="$file.jpg" 226 | fi 227 | 228 | if [ "$extension" == "AAE" ]; then 229 | unsupported 230 | PHOTOS_UNSUPPORTED=$((PHOTOS_UNSUPPORTED+1)) 231 | TO_ADD= 232 | else 233 | # Each photo is tagged with its own SHA1 234 | tags=$(sha1 "$file") 235 | # Prevent local duplicates 236 | sha_lookup "$tags" 237 | if [ "$?" != "0" ]; then 238 | # Prevent remote duplicates 239 | photo_lookup "$tags" 240 | if [ -z "$PHOTO_ID" ]; then 241 | photo_upload "$file" "$tags" 242 | if [ -z "$PHOTO_ID" ]; then 243 | error 244 | PHOTOS_ERROR=$((PHOTOS_ERROR+1)) 245 | TO_ADD= 246 | else 247 | progress 248 | PHOTOS_UPLOADED=$((PHOTOS_UPLOADED+1)) 249 | TO_ADD=1 250 | fi 251 | else 252 | skip 253 | PHOTOS_SKIPPED=$((PHOTOS_SKIPPED+1)) 254 | TO_ADD=1 255 | fi 256 | sha_write "$tags" 257 | else 258 | log "Trying to upload duplicated file: $file. Skipping." 259 | duplicate 260 | PHOTOS_DUPLICATED=$((PHOTOS_DUPLICATED+1)) 261 | TO_ADD= 262 | fi 263 | 264 | if [ "$extension" == "HEIC" ]; then 265 | log "Removing converted file $file" 266 | rm "$file" 267 | fi 268 | fi 269 | } 270 | 271 | function heic_to_jpg { 272 | src=$1 273 | tgt="$1.jpg" 274 | log "Converting $src to $tgt." 275 | magick convert "$src" "$tgt" 276 | } 277 | 278 | function log_init { 279 | mkdir -p "$logdir" 280 | logfile="$logdir/$(datetime)" 281 | echo "$logfile" 282 | } 283 | 284 | function log { 285 | text=$* 286 | prefix=$(datetime) 287 | echo "[$prefix] $text" >> "$logfile" 288 | } 289 | 290 | function sha_write { 291 | sha=$1 292 | echo "$sha" >> "$logfile.sha" 293 | } 294 | 295 | function sha_lookup { 296 | sha=$1 297 | grep -q -s -w "$sha" "$logfile.sha" 298 | } 299 | 300 | ################################################################################ 301 | # Main 302 | ################################################################################ 303 | 304 | # Verify requirements are met 305 | require_program "flickcurl" "http://librdf.org/flickcurl" 306 | 307 | # The script requires exactly two arguments 308 | if [ "$#" != "2" ]; then 309 | usage 310 | exit 1 311 | fi 312 | 313 | # Handle script arguments 314 | directory=$1 315 | album=$2 316 | 317 | # Initialize logging 318 | logfile=$(log_init) 319 | 320 | # Log script name and arguments 321 | log "$(line)" 322 | log "$0 $*" 323 | log "$(line)" 324 | 325 | # Retrieve the list of files to backup 326 | files=($directory/*) 327 | num_files="${#files[@]}" 328 | log "Found $num_files file(s) in directory: $directory" 329 | log "$(line)" 330 | 331 | # To create an album, we require at least one photo 332 | primary="${files:0}" 333 | log "Select primary photo: $primary" 334 | photo_upload_nodup "$primary" 335 | 336 | # Create the album 337 | photoset_create "$album" 338 | 339 | # Upload the rest of the photos 340 | for f in "${files[@]:1}"; do 341 | log "$(line)" 342 | log "Select photo: $f" 343 | photo_upload_nodup "$f" 344 | if ! [ -z $TO_ADD ]; then 345 | photoset_add "$PHOTOSET_ID" "$PHOTO_ID" 346 | fi 347 | done 348 | 349 | # Log summary 350 | log "$(line)" 351 | log "Uploaded $PHOTOS_UPLOADED photos" 352 | log "Skipped $PHOTOS_SKIPPED photos (already uploaded)" 353 | log "Skipped $PHOTOS_DUPLICATED photos (local duplicates)" 354 | log "Skipped $PHOTOS_UNSUPPORTED files (.AAE)" 355 | log "Failed uploading $PHOTOS_ERROR photos" 356 | # Give a bit of time to fetch the proper photos size 357 | sleep 5 358 | num_photos=$(photoset_size) 359 | log "Photoset: $PHOTOSET_ID contains $num_photos photos" 360 | excluded=$(($PHOTOS_DUPLICATED+$PHOTOS_UNSUPPORTED)) 361 | expected=$(($num_files-$excluded)) 362 | if [ "$num_photos" == "$num_files" ]; then 363 | log "Photoset verification succeeded: $num_files - $excluded files == $num_photos photos" 364 | result "OK" 0 365 | else 366 | log "Photoset verification failed: $num_files - $excluded files VS $num_photos photos" 367 | result "ERROR" 1 368 | fi 369 | --------------------------------------------------------------------------------