To provide more context on the issue's origin, we provide {@code mode} as
8 | * an additional constructor field.
9 | */
10 | public class MigrationException extends RuntimeException {
11 | private final MigratorMode mode;
12 |
13 | public MigrationException(String message, MigratorMode mode) {
14 | super(message);
15 | this.mode = mode;
16 | }
17 |
18 | public MigratorMode getMode() {
19 | return mode;
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/resources/s3-example.properties:
--------------------------------------------------------------------------------
1 | # Migrator mode
2 | migrator.mode=S3
3 |
4 | # Source path on local to visit photos
5 | path.source=Dummy/Source
6 |
7 | # Target bucket on S3 to upload photos
8 | s3.bucket=your-bucket-name
9 |
10 | # AWS access key ID
11 | s3.accessKey=Replace with your access key
12 |
13 | # AWS secret access key
14 | s3.secretKey=Replace with your secret key
15 |
16 | # AWS region (optional, defaults to us-east-1)
17 | s3.region=us-east-1
18 |
19 | # Enable delta migration to skip unchanged files (default: false)
20 | # When enabled, only new or modified files will be migrated
21 | # State file is stored locally at source path for cloud destinations
22 | # delta.enabled=true
23 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .gradle
2 | build/
3 | !gradle/wrapper/gradle-wrapper.jar
4 | !**/src/main/**/build/
5 | !**/src/test/**/build/
6 |
7 | ### IntelliJ IDEA ###
8 | .idea/modules.xml
9 | .idea/jarRepositories.xml
10 | .idea/compiler.xml
11 | .idea/libraries/
12 | *.iws
13 | *.iml
14 | *.ipr
15 | out/
16 | !**/src/main/**/out/
17 | !**/src/test/**/out/
18 |
19 | ### Eclipse ###
20 | .apt_generated
21 | .classpath
22 | .factorypath
23 | .project
24 | .settings
25 | .springBeans
26 | .sts4-cache
27 | bin/
28 | !**/src/main/**/bin/
29 | !**/src/test/**/bin/
30 |
31 | ### NetBeans ###
32 | /nbproject/private/
33 | /nbbuild/
34 | /dist/
35 | /nbdist/
36 | /.nb-gradle/
37 |
38 | ### VS Code ###
39 | .vscode/
40 |
41 | ### Mac OS ###
42 | .DS_Store
43 |
--------------------------------------------------------------------------------
/.idea/checkstyle-idea.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 10.20.1
5 | JavaOnly
6 |
7 |
8 |
14 |
15 |
--------------------------------------------------------------------------------
/config/checkstyle/checkstyle.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/MigratorMode.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | /**
4 | * This mode determines whether photos are migrated between local folders or
5 | * migrated to a cloud service.
6 | *
7 | *
Please add the necessary fields to your properties file based on
8 | * the selected mode. Check sample files in {@code src/main/resources}
9 | * for reference.
10 | */
11 | public enum MigratorMode {
12 | /** Migrate photos to a local directory path. */
13 | PATH,
14 |
15 | /** Migrate photos to a Dropbox account. */
16 | DROPBOX,
17 |
18 | /** Migrate photos to a Google Drive account. */
19 | GOOGLE_DRIVE,
20 |
21 | /** Migrate photos to an SFTP server. */
22 | SFTP,
23 |
24 | /** Migrate photos to an Amazon S3 bucket. */
25 | S3
26 | }
27 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on:
4 | push:
5 | branches: [main]
6 | pull_request:
7 | branches: [main]
8 |
9 | permissions:
10 | contents: read
11 |
12 | jobs:
13 | java-build:
14 | name: Java LTS
15 | runs-on: ubuntu-latest
16 |
17 | strategy:
18 | matrix:
19 | java-version: [21, 22, 23, 24, 25]
20 |
21 | steps:
22 | - name: Checkout code
23 | uses: actions/checkout@v5
24 | - name: Set up Java LTS
25 | uses: actions/setup-java@v5
26 | with:
27 | java-version: ${{ matrix.java-version }}
28 | distribution: 'zulu'
29 | cache: 'gradle'
30 | - name: Build source with Gradle
31 | run: gradle build
32 | - name: Upload coverage data to Codecov
33 | uses: codecov/codecov-action@v5
34 | env:
35 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
36 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/traversal/PathRuleSet.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.traversal;
2 |
3 | import org.jetbrains.annotations.NotNull;
4 |
5 | import java.nio.file.Files;
6 | import java.nio.file.Path;
7 | import java.util.List;
8 | import java.util.function.Predicate;
9 |
10 | public record PathRuleSet(List> rules) {
11 | @NotNull
12 | public static PathRuleSet getDefault() {
13 | return new PathRuleSet(List.of(
14 | Files::isRegularFile,
15 | PathRule.isPublic(),
16 | PathRule.validExtensions().or(PathRule.isImageContent()),
17 | PathRule.minimumBytes(100L)));
18 | }
19 |
20 | public boolean matches(Path path) {
21 | return rules.stream().allMatch(rule -> rule.test(path));
22 | }
23 |
24 | public int size() {
25 | return rules.size();
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/resolution/TestResolutionAbstract.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.resolution;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import org.jspecify.annotations.NonNull;
5 |
6 | import java.nio.file.Path;
7 | import java.util.List;
8 |
9 | import static io.huangsam.photohaul.TestHelper.getStaticResources;
10 |
11 | public abstract class TestResolutionAbstract {
12 | private static final Photo BAUER_PHOTO = buildBauerPhoto();
13 |
14 | private static @NonNull Photo buildBauerPhoto() {
15 | Path photoPath = getStaticResources().resolve("bauerlite.jpg");
16 | return new Photo(photoPath);
17 | }
18 |
19 | @NonNull Photo getBauerPhoto() {
20 | return BAUER_PHOTO;
21 | }
22 |
23 | @NonNull PhotoResolver getPhotoResolver() {
24 | return new PhotoResolver(List.of(Photo::make, PhotoFunction.yearTaken()));
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/Main.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul;
2 |
3 | import io.huangsam.photohaul.deduplication.PhotoDeduplicator;
4 | import io.huangsam.photohaul.migration.MigratorFactory;
5 | import io.huangsam.photohaul.resolution.PhotoResolver;
6 | import io.huangsam.photohaul.traversal.PathRuleSet;
7 | import io.huangsam.photohaul.traversal.PhotoCollector;
8 |
9 | public class Main {
10 | public static void main(String[] args) {
11 | Settings settings = Settings.getDefault();
12 | PhotoCollector photoCollector = new PhotoCollector();
13 | PathRuleSet pathRuleSet = PathRuleSet.getDefault();
14 | PhotoDeduplicator deduplicator = new PhotoDeduplicator();
15 | PhotoResolver photoResolver = PhotoResolver.getDefault();
16 | MigratorFactory migratorFactory = new MigratorFactory();
17 |
18 | Application app = new Application(settings, photoCollector, pathRuleSet, deduplicator, photoResolver, migratorFactory);
19 | app.run();
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024 Samuel Huang
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/CREDITS.md:
--------------------------------------------------------------------------------
1 | # Photo credits
2 |
3 | Ensure that all photos used in this project are credited properly.
4 |
5 | ## README photos
6 |
7 | `images/sunny-bunny-tidy-up.webp`
8 |
9 | - Domain:
10 | - Link:
11 | - Author: Sunny Bunnies
12 | - Author ID: [SunnyBunniesOfficial](https://giphy.com/SunnyBunniesOfficial)
13 |
14 | ## Test photos
15 |
16 | `src/test/resources/static/school.png`
17 |
18 | - Domain:
19 | - Link:
20 | - Author Name: Freepik
21 | - Author ID: [freepik](https://www.flaticon.com/authors/freepik)
22 |
23 | `src/test/resources/static/salad.jpg`
24 |
25 | - Domain:
26 | - Link:
27 | - Author Name: Nadine Primeau
28 | - Author ID: [nadineprimeau](https://unsplash.com/@nadineprimeau)
29 |
30 | `src/test/resources/static/bauerlite.jpg`
31 |
32 | - Domain:
33 | - Author: Samuel Huang
34 | - Author ID: [huangsam](https://github.com/huangsam)
35 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/migration/TestMigrationAbstract.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.traversal.PhotoCollector;
4 | import org.jetbrains.annotations.NotNull;
5 | import org.jspecify.annotations.NonNull;
6 |
7 | import java.nio.file.Path;
8 | import java.util.List;
9 |
10 | import static io.huangsam.photohaul.TestHelper.getStaticResources;
11 |
12 | public abstract class TestMigrationAbstract {
13 | void run(@NonNull Migrator migrator) {
14 | run(migrator, List.of("bauerlite.jpg", "salad.jpg"));
15 | }
16 |
17 | void run(@NotNull Migrator migrator, @NonNull List names) {
18 | PhotoCollector photoCollector = getPathCollector(getStaticResources(), names);
19 | migrator.migratePhotos(photoCollector.getPhotos());
20 | }
21 |
22 | @NotNull
23 | private PhotoCollector getPathCollector(@NonNull Path path, @NotNull List names) {
24 | PhotoCollector photoCollector = new PhotoCollector();
25 | for (String name : names) {
26 | photoCollector.addPhoto(path.resolve(name));
27 | }
28 | return photoCollector;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/state/StateFileStorage.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration.state;
2 |
3 | import org.jspecify.annotations.Nullable;
4 |
5 | import java.io.IOException;
6 |
7 | /**
8 | * Interface for reading and writing migration state files.
9 | *
10 | *
Implementations of this interface provide storage-specific operations
11 | * for managing the migration state file at different destinations
12 | * (local filesystem, S3, Dropbox, Google Drive, SFTP).
13 | */
14 | public interface StateFileStorage {
15 | /**
16 | * Read the content of the state file.
17 | *
18 | * @param fileName the name of the state file
19 | * @return the content of the state file, or null if it doesn't exist
20 | * @throws IOException if an I/O error occurs during reading
21 | */
22 | @Nullable String readStateFile(String fileName) throws IOException;
23 |
24 | /**
25 | * Write content to the state file.
26 | *
27 | * @param fileName the name of the state file
28 | * @param content the content to write
29 | * @throws IOException if an I/O error occurs during writing
30 | */
31 | void writeStateFile(String fileName, String content) throws IOException;
32 | }
33 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/traversal/TestPathWalker.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.traversal;
2 |
3 | import org.junit.jupiter.api.Test;
4 |
5 | import java.util.List;
6 |
7 | import static io.huangsam.photohaul.TestHelper.getStaticResources;
8 | import static org.junit.jupiter.api.Assertions.assertFalse;
9 | import static org.junit.jupiter.api.Assertions.assertTrue;
10 |
11 | public class TestPathWalker {
12 | @Test
13 | void testTraverseWithPhotos() {
14 | PhotoCollector photoCollector = new PhotoCollector();
15 | PathRuleSet pathRuleSet = new PathRuleSet(List.of(PathRule.validExtensions()));
16 | PathWalker pathWalker = new PathWalker(getStaticResources(), pathRuleSet);
17 | pathWalker.traverse(photoCollector);
18 | assertFalse(photoCollector.getPhotos().isEmpty());
19 | }
20 |
21 | @Test
22 | void testTraverseWithNoPhotos() {
23 | PhotoCollector photoCollector = new PhotoCollector();
24 | PathRuleSet pathRuleSet = new PathRuleSet(List.of(PathRule.minimumBytes(100_000_000L)));
25 | PathWalker pathWalker = new PathWalker(getStaticResources(), pathRuleSet);
26 | pathWalker.traverse(photoCollector);
27 | assertTrue(photoCollector.getPhotos().isEmpty());
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/Migrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import org.jetbrains.annotations.NotNull;
5 |
6 | import java.util.Collection;
7 |
8 | /**
9 | * Defines the contract for a photo migration service.
10 | *
11 | *
This contract allows consumers to migrate photos without knowing whether
12 | * the operations happen locally or in the cloud.
13 | */
14 | public interface Migrator extends AutoCloseable {
15 | /**
16 | * Migrate a collection of photos to a specific target.
17 | *
18 | *
For each photo record, do the following:
19 | *
20 | *
This class captures the essential metadata used to determine if a file
11 | * has changed since its last migration: the path, file size in bytes,
12 | * and last modified timestamp in milliseconds since epoch.
13 | */
14 | public record FileState(
15 | @NotNull String path,
16 | long size,
17 | long lastModifiedMillis
18 | ) {
19 | /**
20 | * Constructs a FileState with validation.
21 | *
22 | * @param path path of the file
23 | * @param size file size in bytes
24 | * @param lastModifiedMillis last modified timestamp in milliseconds since epoch
25 | * @throws IllegalArgumentException if path is blank, size is negative, or lastModifiedMillis is negative
26 | */
27 | public FileState {
28 | Objects.requireNonNull(path, "Path cannot be null");
29 | if (path.isBlank()) {
30 | throw new IllegalArgumentException("Path cannot be blank");
31 | }
32 | if (size < 0) {
33 | throw new IllegalArgumentException("Size cannot be negative");
34 | }
35 | if (lastModifiedMillis < 0) {
36 | throw new IllegalArgumentException("Last modified timestamp cannot be negative");
37 | }
38 | }
39 |
40 | /**
41 | * Check if this file state matches another file's current state.
42 | *
43 | * @param other the other file state to compare
44 | * @return true if size and lastModifiedMillis match
45 | */
46 | public boolean matches(@NotNull FileState other) {
47 | return this.size == other.size && this.lastModifiedMillis == other.lastModifiedMillis;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/resolution/TestPhotoResolver.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.resolution;
2 |
3 | import org.junit.jupiter.api.Test;
4 |
5 | import java.util.List;
6 |
7 | import static org.junit.jupiter.api.Assertions.assertEquals;
8 | import static org.junit.jupiter.api.Assertions.assertThrows;
9 | import static org.junit.jupiter.api.Assertions.assertTrue;
10 |
11 | public class TestPhotoResolver extends TestResolutionAbstract {
12 | @Test
13 | void testResolveListOnMakeYear() {
14 | List resolvedList = getPhotoResolver().resolveList(getBauerPhoto());
15 | assertEquals(2, resolvedList.size());
16 | assertEquals("Canon", resolvedList.getFirst());
17 | assertEquals("2023", resolvedList.get(1));
18 | }
19 |
20 | @Test
21 | void testResolveStringOnMakeYearWithDefaultDelimiter() {
22 | String resolvedString = getPhotoResolver().resolveString(getBauerPhoto());
23 | assertEquals("Canon/2023", resolvedString);
24 | }
25 |
26 | @Test
27 | void testResolveStringOnMakeYearWithCustomDelimiter() {
28 | String resolvedString = getPhotoResolver().resolveString(getBauerPhoto(), " - ");
29 | assertEquals("Canon - 2023", resolvedString);
30 | }
31 |
32 | @Test
33 | void testDefaultResolverIsNotEmpty() {
34 | PhotoResolver defaultResolver = PhotoResolver.getDefault();
35 | assertTrue(defaultResolver.size() > 0);
36 | }
37 |
38 | @Test
39 | void testEmptyResolverIsEmpty() {
40 | PhotoResolver emptyResolver = new PhotoResolver(List.of());
41 | List resolvedList = emptyResolver.resolveList(getBauerPhoto());
42 | assertTrue(resolvedList.isEmpty());
43 | }
44 |
45 | @Test
46 | void testResolverThrowsException() {
47 | PhotoResolver faultyResolver = new PhotoResolver(List.of(photo -> null));
48 | assertThrows(ResolutionException.class, () -> faultyResolver.resolveList(getBauerPhoto()));
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/traversal/TestPathRule.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.traversal;
2 |
3 | import org.junit.jupiter.api.Test;
4 |
5 | import java.nio.file.Path;
6 |
7 | import static io.huangsam.photohaul.TestHelper.getStaticResources;
8 | import static org.junit.jupiter.api.Assertions.assertFalse;
9 | import static org.junit.jupiter.api.Assertions.assertTrue;
10 |
11 | public class TestPathRule {
12 | private static final Path REAL_PATH = getStaticResources().resolve("bauerlite.jpg");
13 | private static final Path FAKE_PATH = getStaticResources().resolve("bauerlite.foo");
14 | private static final Path HIDDEN_PATH = getStaticResources().resolve(".bauerlite.hidden");
15 |
16 | @Test
17 | void testIsValidExtensionWithRealIsTrue() {
18 | assertTrue(PathRule.validExtensions().test(REAL_PATH));
19 | }
20 |
21 | @Test
22 | void testIsValidExtensionWithFakeIsFalse() {
23 | assertFalse(PathRule.validExtensions().test(FAKE_PATH));
24 | }
25 |
26 | @Test
27 | void testIsImageContentWithRealIsTrue() {
28 | assertTrue(PathRule.isImageContent().test(REAL_PATH));
29 | }
30 |
31 | @Test
32 | void testIsImageContentWithFakeIsFalse() {
33 | assertFalse(PathRule.isImageContent().test(FAKE_PATH));
34 | }
35 |
36 | @Test
37 | void testIsMinimumBytesWithRealIsTrue() {
38 | assertTrue(PathRule.minimumBytes(100L).test(REAL_PATH));
39 | }
40 |
41 | @Test
42 | void testIsMinimumBytesWithFakeIsFalse() {
43 | assertFalse(PathRule.minimumBytes(100L).test(FAKE_PATH));
44 | }
45 |
46 | @Test
47 | void testIsPublicWithRealIsTrue() {
48 | assertTrue(PathRule.isPublic().test(REAL_PATH));
49 | }
50 |
51 | @Test
52 | void testIsPublicWithFakeIsTrue() {
53 | assertTrue(PathRule.isPublic().test(FAKE_PATH));
54 | }
55 |
56 | @Test
57 | void testIsPublicWithHiddenIsFalse() {
58 | assertFalse(PathRule.isPublic().test(HIDDEN_PATH));
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/Application.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul;
2 |
3 | import io.huangsam.photohaul.deduplication.PhotoDeduplicator;
4 | import io.huangsam.photohaul.migration.MigrationException;
5 | import io.huangsam.photohaul.migration.MigratorFactory;
6 | import io.huangsam.photohaul.migration.MigratorMode;
7 | import io.huangsam.photohaul.model.Photo;
8 | import io.huangsam.photohaul.resolution.PhotoResolver;
9 | import io.huangsam.photohaul.migration.Migrator;
10 | import io.huangsam.photohaul.traversal.PathRuleSet;
11 | import io.huangsam.photohaul.traversal.PathWalker;
12 | import io.huangsam.photohaul.traversal.PhotoCollector;
13 | import org.slf4j.Logger;
14 |
15 | import java.util.Collection;
16 |
17 | import static org.slf4j.LoggerFactory.getLogger;
18 |
19 | public record Application(Settings settings,
20 | PhotoCollector photoCollector, PathRuleSet pathRuleSet,
21 | PhotoDeduplicator deduplicator, PhotoResolver photoResolver,
22 | MigratorFactory migratorFactory) {
23 | private static final Logger LOG = getLogger(Application.class);
24 |
25 | public void run() {
26 | PathWalker pathWalker = new PathWalker(settings.getSourcePath(), pathRuleSet);
27 | pathWalker.traverse(photoCollector);
28 |
29 | Collection uniquePhotos = deduplicator.deduplicate(photoCollector.getPhotos());
30 |
31 | MigratorMode migratorMode = settings.getMigratorMode();
32 |
33 | try (Migrator migrator = migratorFactory.make(migratorMode, settings, photoResolver)) {
34 | migrator.migratePhotos(uniquePhotos);
35 | LOG.info("Finish with success={} failure={}", migrator.getSuccessCount(), migrator.getFailureCount());
36 | } catch (MigrationException e) {
37 | LOG.error("Cannot migrate with mode {}: {}", e.getMode(), e.getMessage());
38 | } catch (Exception e) {
39 | LOG.error("Error during migration or closing migrator: {}", e.getMessage());
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/gradle/libs.versions.toml:
--------------------------------------------------------------------------------
1 | [versions]
2 | sshj = "0.40.0"
3 | dropbox-sdk = "7.0.0"
4 | google-api-client = "2.8.1"
5 | google-api-drive = "v3-rev20251210-2.0.0"
6 | google-auth-library = "1.41.0"
7 | gson = "2.13.2"
8 | jetbrains-annotations = "26.0.2-1"
9 | junit-bom = "6.0.1"
10 | logback-classic = "1.5.22"
11 | metadata-extractor = "2.19.0"
12 | mockito = "5.21.0"
13 | slf4j-api = "2.0.17"
14 | task-tree = "4.0.1"
15 | aws-sdk-s3 = "2.40.8"
16 |
17 | [libraries]
18 | sshj = { module = "com.hierynomus:sshj", version.ref = "sshj" }
19 | dropbox-sdk = { module = "com.dropbox.core:dropbox-core-sdk", version.ref = "dropbox-sdk" }
20 | google-api-client = { module = "com.google.api-client:google-api-client", version.ref = "google-api-client" }
21 | google-api-drive = { module = "com.google.apis:google-api-services-drive", version.ref = "google-api-drive" }
22 | google-auth-library = { module = "com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth-library" }
23 | gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
24 | jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "jetbrains-annotations" }
25 | junit-bom = { module = "org.junit:junit-bom", version.ref = "junit-bom" }
26 | junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" }
27 | junit-launcher = { module = "org.junit.platform:junit-platform-launcher" }
28 | logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback-classic" }
29 | metadata-extractor = { module = "com.drewnoakes:metadata-extractor", version.ref = "metadata-extractor"}
30 | mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" }
31 | mockito-junit = { module = "org.mockito:mockito-junit-jupiter", version.ref = "mockito" }
32 | slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j-api" }
33 | aws-sdk-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws-sdk-s3" }
34 |
35 | [plugins]
36 | task-tree = { id = "com.dorongold.task-tree", version.ref = "task-tree" }
37 |
38 | [bundles]
39 | mockito-all = ["mockito-core", "mockito-junit"]
40 | google-all = ["google-api-client", "google-api-drive", "google-auth-library", "gson"]
41 | aws-all = ["aws-sdk-s3"]
42 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/S3Migrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import io.huangsam.photohaul.resolution.PhotoResolver;
5 | import io.huangsam.photohaul.resolution.ResolutionException;
6 | import org.jetbrains.annotations.NotNull;
7 | import org.jspecify.annotations.NonNull;
8 | import org.slf4j.Logger;
9 | import software.amazon.awssdk.services.s3.S3Client;
10 | import software.amazon.awssdk.services.s3.model.PutObjectRequest;
11 |
12 | import java.util.Collection;
13 |
14 | import static org.slf4j.LoggerFactory.getLogger;
15 |
16 | public class S3Migrator implements Migrator {
17 | private static final Logger LOG = getLogger(S3Migrator.class);
18 |
19 | private final @NonNull String bucketName;
20 | private final PhotoResolver photoResolver;
21 | private final S3Client s3Client;
22 |
23 | private long successCount = 0L;
24 | private long failureCount = 0L;
25 |
26 | public S3Migrator(@NotNull String bucket, PhotoResolver resolver, S3Client client) {
27 | bucketName = bucket;
28 | photoResolver = resolver;
29 | s3Client = client;
30 | }
31 |
32 | @Override
33 | public void migratePhotos(@NotNull Collection photos) {
34 | LOG.debug("Start S3 migration to bucket {}", bucketName);
35 | photos.forEach(photo -> {
36 | String key = getTargetKey(photo);
37 | LOG.trace("Upload {} to s3://{}/{}", photo.name(), bucketName, key);
38 | try {
39 | PutObjectRequest request = PutObjectRequest.builder()
40 | .bucket(bucketName)
41 | .key(key)
42 | .build();
43 | s3Client.putObject(request, photo.path());
44 | successCount++;
45 | } catch (Exception e) {
46 | LOG.error("Cannot upload {}: {}", photo.name(), e.getMessage());
47 | failureCount++;
48 | }
49 | });
50 | }
51 |
52 | @Override
53 | public long getSuccessCount() {
54 | return successCount;
55 | }
56 |
57 | @Override
58 | public long getFailureCount() {
59 | return failureCount;
60 | }
61 |
62 | @Override
63 | public void close() throws Exception {
64 | s3Client.close();
65 | }
66 |
67 | @NotNull
68 | private String getTargetKey(@NonNull Photo photo) {
69 | try {
70 | return photoResolver.resolveString(photo) + "/" + photo.name();
71 | } catch (ResolutionException e) {
72 | return "Other/" + photo.name();
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/migration/TestS3Migrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import io.huangsam.photohaul.resolution.PhotoResolver;
5 | import io.huangsam.photohaul.resolution.ResolutionException;
6 | import org.junit.jupiter.api.Test;
7 | import org.junit.jupiter.api.extension.ExtendWith;
8 | import org.mockito.Mock;
9 | import org.mockito.junit.jupiter.MockitoExtension;
10 | import software.amazon.awssdk.services.s3.S3Client;
11 |
12 | import software.amazon.awssdk.services.s3.model.PutObjectRequest;
13 |
14 | import java.nio.file.Path;
15 |
16 | import static org.junit.jupiter.api.Assertions.assertEquals;
17 | import static org.mockito.ArgumentMatchers.any;
18 | import static org.mockito.Mockito.times;
19 | import static org.mockito.Mockito.verify;
20 | import static org.mockito.Mockito.when;
21 |
22 | @ExtendWith(MockitoExtension.class)
23 | public class TestS3Migrator extends TestMigrationAbstract {
24 | @Mock
25 | S3Client s3ClientMock;
26 |
27 | @Mock
28 | PhotoResolver photoResolverMock;
29 |
30 | @Test
31 | void testMigratePhotosSuccess() throws Exception {
32 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
33 |
34 | Migrator migrator = new S3Migrator("test-bucket", photoResolverMock, s3ClientMock);
35 | run(migrator);
36 |
37 | verify(s3ClientMock, times(2)).putObject(any(PutObjectRequest.class), any(Path.class));
38 |
39 | assertEquals(2, migrator.getSuccessCount());
40 | assertEquals(0, migrator.getFailureCount());
41 |
42 | migrator.close();
43 | verify(s3ClientMock).close();
44 | }
45 |
46 | @Test
47 | void testMigratePhotosUploadFailure() throws Exception {
48 | when(s3ClientMock.putObject(any(PutObjectRequest.class), any(Path.class))).thenThrow(new RuntimeException("Upload failed"));
49 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
50 |
51 | Migrator migrator = new S3Migrator("test-bucket", photoResolverMock, s3ClientMock);
52 | run(migrator);
53 |
54 | verify(s3ClientMock, times(2)).putObject(any(PutObjectRequest.class), any(Path.class));
55 |
56 | assertEquals(0, migrator.getSuccessCount());
57 | assertEquals(2, migrator.getFailureCount());
58 |
59 | migrator.close();
60 | verify(s3ClientMock).close();
61 | }
62 |
63 | @Test
64 | void testMigratePhotosWithResolutionException() throws Exception {
65 | when(photoResolverMock.resolveString(any(Photo.class))).thenThrow(new ResolutionException("Resolution failed"));
66 |
67 | Migrator migrator = new S3Migrator("test-bucket", photoResolverMock, s3ClientMock);
68 | run(migrator);
69 |
70 | verify(s3ClientMock, times(2)).putObject(any(PutObjectRequest.class), any(Path.class));
71 |
72 | assertEquals(2, migrator.getSuccessCount());
73 | assertEquals(0, migrator.getFailureCount());
74 |
75 | migrator.close();
76 | verify(s3ClientMock).close();
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
1 | @rem
2 | @rem Copyright 2015 the original author or authors.
3 | @rem
4 | @rem Licensed under the Apache License, Version 2.0 (the "License");
5 | @rem you may not use this file except in compliance with the License.
6 | @rem You may obtain a copy of the License at
7 | @rem
8 | @rem https://www.apache.org/licenses/LICENSE-2.0
9 | @rem
10 | @rem Unless required by applicable law or agreed to in writing, software
11 | @rem distributed under the License is distributed on an "AS IS" BASIS,
12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | @rem See the License for the specific language governing permissions and
14 | @rem limitations under the License.
15 | @rem
16 | @rem SPDX-License-Identifier: Apache-2.0
17 | @rem
18 |
19 | @if "%DEBUG%"=="" @echo off
20 | @rem ##########################################################################
21 | @rem
22 | @rem Gradle startup script for Windows
23 | @rem
24 | @rem ##########################################################################
25 |
26 | @rem Set local scope for the variables with windows NT shell
27 | if "%OS%"=="Windows_NT" setlocal
28 |
29 | set DIRNAME=%~dp0
30 | if "%DIRNAME%"=="" set DIRNAME=.
31 | @rem This is normally unused
32 | set APP_BASE_NAME=%~n0
33 | set APP_HOME=%DIRNAME%
34 |
35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter.
36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
37 |
38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
40 |
41 | @rem Find java.exe
42 | if defined JAVA_HOME goto findJavaFromJavaHome
43 |
44 | set JAVA_EXE=java.exe
45 | %JAVA_EXE% -version >NUL 2>&1
46 | if %ERRORLEVEL% equ 0 goto execute
47 |
48 | echo. 1>&2
49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
50 | echo. 1>&2
51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2
52 | echo location of your Java installation. 1>&2
53 |
54 | goto fail
55 |
56 | :findJavaFromJavaHome
57 | set JAVA_HOME=%JAVA_HOME:"=%
58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
59 |
60 | if exist "%JAVA_EXE%" goto execute
61 |
62 | echo. 1>&2
63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
64 | echo. 1>&2
65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2
66 | echo location of your Java installation. 1>&2
67 |
68 | goto fail
69 |
70 | :execute
71 | @rem Setup the command line
72 |
73 |
74 |
75 | @rem Execute Gradle
76 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
77 |
78 | :end
79 | @rem End local scope for the variables with windows NT shell
80 | if %ERRORLEVEL% equ 0 goto mainEnd
81 |
82 | :fail
83 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
84 | rem the _cmd.exe /c_ return code!
85 | set EXIT_CODE=%ERRORLEVEL%
86 | if %EXIT_CODE% equ 0 set EXIT_CODE=1
87 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
88 | exit /b %EXIT_CODE%
89 |
90 | :mainEnd
91 | if "%OS%"=="Windows_NT" endlocal
92 |
93 | :omega
94 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/migration/TestPathMigrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import io.huangsam.photohaul.resolution.PhotoResolver;
5 | import io.huangsam.photohaul.resolution.ResolutionException;
6 | import org.jetbrains.annotations.NotNull;
7 | import org.junit.jupiter.api.Test;
8 | import org.junit.jupiter.api.extension.ExtendWith;
9 | import org.mockito.Mock;
10 | import org.mockito.junit.jupiter.MockitoExtension;
11 |
12 | import java.nio.file.Path;
13 | import java.util.List;
14 |
15 | import static io.huangsam.photohaul.TestHelper.getTempResources;
16 | import static org.junit.jupiter.api.Assertions.assertEquals;
17 | import static org.mockito.ArgumentMatchers.any;
18 | import static org.mockito.Mockito.when;
19 |
20 | @ExtendWith(MockitoExtension.class)
21 | public class TestPathMigrator extends TestMigrationAbstract {
22 | @Mock
23 | PhotoResolver photoResolverMock;
24 |
25 | @Test
26 | void testMigratePhotosDryRunAllSuccess() throws Exception {
27 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
28 |
29 | Migrator migrator = getPathMover(getTempResources(), photoResolverMock, PathMigrator.Action.DRY_RUN);
30 | run(migrator);
31 |
32 | assertEquals(2, migrator.getSuccessCount());
33 | assertEquals(0, migrator.getFailureCount());
34 |
35 | migrator.close(); // No-op
36 | }
37 |
38 | @Test
39 | void testMigratePhotosCopyAllSuccess() throws Exception {
40 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
41 |
42 | Migrator migrator = getPathMover(getTempResources(), photoResolverMock, PathMigrator.Action.COPY);
43 | run(migrator);
44 |
45 | assertEquals(2, migrator.getSuccessCount());
46 | assertEquals(0, migrator.getFailureCount());
47 |
48 | migrator.close(); // No-op
49 | }
50 |
51 | @Test
52 | void testMigratePhotosMoveAllFailure() throws Exception {
53 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
54 |
55 | Migrator migrator = getPathMover(getTempResources(), photoResolverMock, PathMigrator.Action.MOVE);
56 | run(migrator, List.of("foobar.jpg"));
57 |
58 | assertEquals(0, migrator.getSuccessCount());
59 | assertEquals(1, migrator.getFailureCount());
60 |
61 | migrator.close(); // No-op
62 | }
63 |
64 | @Test
65 | void testMigratePhotosWithResolutionException() throws Exception {
66 | when(photoResolverMock.resolveString(any(Photo.class))).thenThrow(new ResolutionException("Resolution failed"));
67 |
68 | Migrator migrator = getPathMover(getTempResources(), photoResolverMock, PathMigrator.Action.COPY);
69 | run(migrator);
70 |
71 | assertEquals(2, migrator.getSuccessCount());
72 | assertEquals(0, migrator.getFailureCount());
73 |
74 | migrator.close(); // No-op
75 | }
76 |
77 | @NotNull
78 | private static PathMigrator getPathMover(Path destination, PhotoResolver resolver, PathMigrator.Action action) {
79 | return new PathMigrator(destination, resolver, action);
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/DropboxMigrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import com.dropbox.core.DbxException;
4 | import com.dropbox.core.v2.DbxClientV2;
5 | import com.dropbox.core.v2.files.DbxUserFilesRequests;
6 | import com.dropbox.core.v2.files.ListFolderErrorException;
7 | import io.huangsam.photohaul.model.Photo;
8 | import io.huangsam.photohaul.resolution.PhotoResolver;
9 | import io.huangsam.photohaul.resolution.ResolutionException;
10 | import org.jetbrains.annotations.NotNull;
11 | import org.jspecify.annotations.NonNull;
12 | import org.slf4j.Logger;
13 |
14 | import java.io.IOException;
15 | import java.io.InputStream;
16 | import java.nio.file.Files;
17 | import java.util.Collection;
18 |
19 | import static org.slf4j.LoggerFactory.getLogger;
20 |
21 | public class DropboxMigrator implements Migrator {
22 | private static final Logger LOG = getLogger(DropboxMigrator.class);
23 |
24 | private final @NonNull String targetRoot;
25 | private final PhotoResolver photoResolver;
26 | private final DbxClientV2 dropboxClient;
27 |
28 | private long successCount = 0L;
29 | private long failureCount = 0L;
30 |
31 | public DropboxMigrator(@NotNull String target, PhotoResolver resolver, DbxClientV2 client) {
32 | if (!target.startsWith("/")) {
33 | throw new IllegalArgumentException("Target must begin with a '/' character");
34 | }
35 | targetRoot = target;
36 | photoResolver = resolver;
37 | dropboxClient = client;
38 | }
39 |
40 | @Override
41 | public void migratePhotos(@NotNull Collection photos) {
42 | LOG.debug("Start Dropbox migration to {}", targetRoot);
43 | DbxUserFilesRequests requests = dropboxClient.files();
44 | photos.forEach(photo -> {
45 | String targetPath = getTargetPath(photo);
46 | LOG.trace("Move {} to {}", photo.name(), targetPath);
47 | try (InputStream in = Files.newInputStream(photo.path())) {
48 | try {
49 | requests.listFolder(targetPath);
50 | } catch (ListFolderErrorException e) {
51 | requests.createFolderV2(targetPath);
52 | }
53 | requests.uploadBuilder(targetPath + "/" + photo.name()).uploadAndFinish(in);
54 | successCount++;
55 | } catch (IOException | DbxException e) {
56 | LOG.error("Cannot move {}: {}", photo.name(), e.getMessage());
57 | failureCount++;
58 | }
59 | });
60 | }
61 |
62 | @Override
63 | public long getSuccessCount() {
64 | return successCount;
65 | }
66 |
67 | @Override
68 | public long getFailureCount() {
69 | return failureCount;
70 | }
71 |
72 | @Override
73 | public void close() throws Exception {
74 | // No-op: DbxClientV2 does not provide a close method
75 | }
76 |
77 | @NotNull
78 | private String getTargetPath(Photo photo) {
79 | try {
80 | return targetRoot + "/" + photoResolver.resolveString(photo);
81 | } catch (ResolutionException e) {
82 | return targetRoot + "/Other";
83 | }
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Photohaul
2 |
3 | [](https://github.com/huangsam/photohaul/actions)
4 | [](https://codecov.io/gh/huangsam/photohaul)
5 | [](https://github.com/huangsam/photohaul/blob/main/LICENSE)
6 | [](https://github.com/huangsam/photohaul/releases/latest)
7 |
8 | Effortless photo management.
9 |
10 | - Reorganize 10K+ photos (30 GB) in seconds!
11 | - Migrate photos locally and to the cloud (Dropbox, Google Drive, SFTP, S3)
12 | - Customize folder structures based on date, camera, and more
13 | - Filter photos by file type, size, and other criteria
14 | - Detect photo duplicates using modern hash techniques
15 | - Skip unchanged files for faster subsequent runs
16 |
17 | Say goodbye to photo clutter - 👋 + 🚀
18 |
19 | 
20 |
21 | ## Motivation
22 |
23 | As an avid photographer, I use Adobe Lightroom to organize my edited SLR
24 | photos with custom file names and folder structures. This has worked well
25 | for me since 2015, when I started getting serious about photography.
26 |
27 | I want to apply those same patterns to old photos, so that it is easier
28 | for me to access my precious memories. However, I struggle to apply the
29 | same organization to my vast collection of older photos. I cannot apply
30 | Lightroom settings to previously exported images and writing custom
31 | scripts seems daunting.
32 |
33 | I also want to migrate my photos over to a NAS or a cloud provider like
34 | Google Drive, but it involves endless rounds of manual drag-and-drop
35 | operations. I keep thinking to myself - is there a solution out there
36 | that "just works" for my workflow?
37 |
38 | ## Value
39 |
40 | Photohaul addresses the pain points above by providing a central hub for
41 | photographers to filter, organize, and migrate photos to local storage
42 | and cloud services. The folder structure for photos can be based on info
43 | such as year taken and camera make.
44 |
45 | ## Getting started
46 |
47 | For detailed instructions: [link](USERGUIDE.md)
48 |
49 | **Install prerequisites:**
50 |
51 | - Java 21 or later
52 | - Gradle 9 or later
53 |
54 | **Build application:**
55 |
56 | - Run `./gradlew build` in your terminal
57 |
58 | **Configure settings:**
59 |
60 | - Set `PathRuleSet` to filter by extension, file size, etc.
61 | - Set `MigratorMode` to `PATH` / `DROPBOX` / `GOOGLE_DRIVE` / `SFTP` / `S3`
62 | - Set `PhotoResolver` to adjust folder structure
63 | - Fill config file. Refer to examples in [src/main/resources](src/main/resources)
64 |
65 | **Run application:**
66 |
67 | - Run `./gradlew run` in your terminal
68 | - Optional: override with `-Dphotohaul.config=personal/path.properties` (classpath) or an absolute/relative filesystem path
69 | (e.g., `-Dphotohaul.config=./src/main/resources/personal/path.properties`).
70 |
71 | ```text
72 | > Task :run
73 | 08:05:14.518 [main] INFO io.huangsam.photohaul.Settings -- Use config file from photohaul.config: personal/path.properties
74 | 08:05:14.520 [main] INFO io.huangsam.photohaul.Settings -- Loaded settings from classpath: personal/path.properties
75 | 08:05:14.526 [main] DEBUG io.huangsam.photohaul.traversal.PathWalker -- Start traversal of /Users/samhuang/Pictures/Dummy PNG
76 | 08:05:14.536 [main] INFO io.huangsam.photohaul.deduplication.PhotoDeduplicator -- Deduplication complete: 6 unique photos, 0 duplicates removed
77 | 08:05:14.536 [main] DEBUG io.huangsam.photohaul.migration.PathMigrator -- Start path migration to /Users/samhuang/Pictures/Dummy FIN
78 | 08:05:14.627 [main] INFO io.huangsam.photohaul.Application -- Finish with success=6 failure=0
79 |
80 | BUILD SUCCESSFUL in 535ms
81 | 3 actionable tasks: 1 executed, 2 up-to-date
82 | ```
83 |
84 | **That's it!** Sit back and rediscover your memories! 😎 + 🍹 + 🌴
85 |
86 | You're welcome 🙏
87 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/PathMigrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import io.huangsam.photohaul.resolution.PhotoResolver;
5 | import io.huangsam.photohaul.resolution.ResolutionException;
6 | import org.jetbrains.annotations.NotNull;
7 | import org.jspecify.annotations.NonNull;
8 | import org.slf4j.Logger;
9 |
10 | import java.io.IOException;
11 | import java.nio.file.Files;
12 | import java.nio.file.Path;
13 | import java.nio.file.StandardCopyOption;
14 | import java.util.Collection;
15 |
16 | import static org.slf4j.LoggerFactory.getLogger;
17 |
18 | public class PathMigrator implements Migrator {
19 | private static final Logger LOG = getLogger(PathMigrator.class);
20 |
21 | private final Path targetRoot;
22 | private final PhotoResolver photoResolver;
23 | private final Action migratorAction;
24 |
25 | private long successCount = 0L;
26 | private long failureCount = 0L;
27 |
28 | public PathMigrator(Path target, PhotoResolver resolver, Action action) {
29 | targetRoot = target;
30 | photoResolver = resolver;
31 | migratorAction = action;
32 | }
33 |
34 | @Override
35 | public final void migratePhotos(@NotNull Collection photos) {
36 | LOG.debug("Start path migration to {}", targetRoot);
37 | photos.forEach(photo -> {
38 | Path targetPath = getTargetPath(photo);
39 | LOG.trace("Move {} to {}", photo.name(), targetPath);
40 | try {
41 | migratePhoto(targetPath, photo);
42 | successCount++;
43 | } catch (IOException e) {
44 | LOG.error("Cannot move {}: {}", photo.name(), e.getMessage());
45 | failureCount++;
46 | }
47 | });
48 | }
49 |
50 | @Override
51 | public final long getSuccessCount() {
52 | return successCount;
53 | }
54 |
55 | @Override
56 | public final long getFailureCount() {
57 | return failureCount;
58 | }
59 |
60 | @Override
61 | public void close() throws Exception {
62 | // No-op: no resources to close
63 | }
64 |
65 | @NotNull
66 | private Path getTargetPath(Photo photo) {
67 | try {
68 | return targetRoot.resolve(photoResolver.resolveString(photo));
69 | } catch (ResolutionException e) {
70 | return targetRoot.resolve("Other");
71 | }
72 | }
73 |
74 | private void migratePhoto(@NonNull Path target, @NonNull Photo photo) throws IOException {
75 | Path photoLocation = target.resolve(photo.name());
76 | if (migratorAction == Action.DRY_RUN) {
77 | LOG.info("Dry-run {} to {}", photo.path(), photoLocation);
78 | return;
79 | }
80 | Files.createDirectories(target);
81 | switch (migratorAction) {
82 | case MOVE -> Files.move(photo.path(), photoLocation, StandardCopyOption.REPLACE_EXISTING);
83 | case COPY -> Files.copy(photo.path(), photoLocation, StandardCopyOption.REPLACE_EXISTING);
84 | }
85 | }
86 |
87 | /**
88 | * This action can be a {@code mv}, {@code cp} or {@code echo} in Linux speak.
89 | * The {@code echo} op is good to try before settling on other actions.
90 | */
91 | public enum Action {
92 | /**
93 | * Move the photo from its original location to the target path.
94 | * Permanently removes the photo from its original location.
95 | */
96 | MOVE,
97 |
98 | /**
99 | * Copy the photo from its original location to the target path.
100 | * The original photo remains untouched.
101 | */
102 | COPY,
103 |
104 | /**
105 | * Perform a dry run of the migration process.
106 | * No files are actually moved or copied.
107 | * Logs information about where each photo would be placed.
108 | */
109 | DRY_RUN
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/SftpMigrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import io.huangsam.photohaul.model.Photo;
4 | import io.huangsam.photohaul.resolution.PhotoResolver;
5 | import io.huangsam.photohaul.resolution.ResolutionException;
6 | import net.schmizz.sshj.SSHClient;
7 | import net.schmizz.sshj.sftp.SFTPClient;
8 | import org.jetbrains.annotations.NotNull;
9 | import org.jspecify.annotations.NonNull;
10 | import org.slf4j.Logger;
11 |
12 | import java.io.IOException;
13 | import java.nio.file.Path;
14 | import java.nio.file.Paths;
15 | import java.util.function.Supplier;
16 |
17 | import static org.slf4j.LoggerFactory.getLogger;
18 |
19 | public class SftpMigrator implements Migrator {
20 | private static final Logger LOG = getLogger(SftpMigrator.class);
21 |
22 | private final @NonNull String host;
23 | private final int port;
24 | private final @NonNull String username;
25 | private final @NonNull String password;
26 | private final @NonNull String targetRoot;
27 | private final PhotoResolver photoResolver;
28 | private final Supplier sshClientSupplier;
29 |
30 | private long successCount = 0L;
31 | private long failureCount = 0L;
32 |
33 | public SftpMigrator(@NotNull String host, int port, @NotNull String username, @NotNull String password,
34 | @NotNull String target, PhotoResolver resolver) {
35 | this(host, port, username, password, target, resolver, SSHClient::new);
36 | }
37 |
38 | // For testing
39 | SftpMigrator(@NotNull String host, int port, @NotNull String username, @NotNull String password,
40 | @NotNull String target, PhotoResolver resolver, Supplier sshClientSupplier) {
41 | this.host = host;
42 | this.port = port;
43 | this.username = username;
44 | this.password = password;
45 | this.targetRoot = target;
46 | this.photoResolver = resolver;
47 | this.sshClientSupplier = sshClientSupplier;
48 | }
49 |
50 | @Override
51 | public void migratePhotos(@NotNull java.util.Collection photos) {
52 | LOG.debug("Start SFTP migration to {}@{}:{}", username, host, port);
53 | int processedCount = 0;
54 | try (SSHClient sshClient = sshClientSupplier.get()) {
55 | sshClient.loadKnownHosts();
56 | sshClient.connect(host, port);
57 | sshClient.authPassword(username, password);
58 |
59 | try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
60 | for (Photo photo : photos) {
61 | String targetPath = getTargetPath(photo);
62 | LOG.trace("Upload {} to {}", photo.name(), targetPath);
63 | try {
64 | // Ensure target directory exists
65 | Path targetPathObj = Paths.get(targetPath);
66 | Path targetDir = targetPathObj.getParent();
67 | if (targetDir != null) {
68 | sftpClient.mkdirs(targetDir.toString());
69 | }
70 | sftpClient.put(photo.path().toString(), targetPath);
71 | successCount++;
72 | } catch (IOException e) {
73 | LOG.error("Cannot upload {}: {}", photo.name(), e.getMessage());
74 | failureCount++;
75 | }
76 | processedCount++;
77 | }
78 | }
79 | } catch (IOException e) {
80 | LOG.error("SFTP connection error: {}", e.getMessage());
81 | failureCount += (photos.size() - processedCount);
82 | }
83 | }
84 |
85 | @Override
86 | public long getSuccessCount() {
87 | return successCount;
88 | }
89 |
90 | @Override
91 | public long getFailureCount() {
92 | return failureCount;
93 | }
94 |
95 | @Override
96 | public void close() throws Exception {
97 | // No-op: resources are closed within migratePhotos
98 | }
99 |
100 | @NotNull
101 | private String getTargetPath(@NonNull Photo photo) {
102 | try {
103 | return targetRoot + "/" + photoResolver.resolveString(photo) + "/" + photo.name();
104 | } catch (ResolutionException e) {
105 | return targetRoot + "/Other/" + photo.name();
106 | }
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/AGENTS.md:
--------------------------------------------------------------------------------
1 | # Photohaul Codebase Overview for Agents
2 |
3 | ## Project Description
4 |
5 | Photohaul is a Java-based CLI application for migrating photo collections to various destinations (local paths, Dropbox, Google Drive, SFTP, S3) with built-in deduplication to avoid uploading duplicates. It uses SHA-256 hashing for accurate duplicate detection and supports lazy metadata extraction for performance.
6 |
7 | Main Class: `io.huangsam.photohaul.Main`
8 |
9 | Build Tool: Gradle (with application plugin for CLI execution)
10 |
11 | ## Directory Structure (`src/`)
12 |
13 | ### Package `io.huangsam.photohaul`
14 |
15 | - Main entry point for the CLI application.
16 | - Handles argument parsing and configuration loading.
17 | - Orchestrates photo collection, deduplication, and migration processes.
18 |
19 | ### Package `io.huangsam.photohaul.model`
20 |
21 | - Defines core data models for photos.
22 | - Implements lazy-loaded metadata extraction for performance.
23 | - Provides builder patterns for photo instances.
24 |
25 | ### Package `io.huangsam.photohaul.deduplication`
26 |
27 | - Implements multi-level photo deduplication logic.
28 | - Uses size, partial hash, and full SHA-256 hashing for efficient duplicate detection.
29 | - Processes photos in streams for parallel-friendly operations.
30 |
31 | ### Package `io.huangsam.photohaul.migration`
32 |
33 | - Provides migration strategies and implementations.
34 | - Supports transferring photos to local paths, Dropbox, Google Drive, SFTP, and S3 servers.
35 | - Includes factory patterns for configurability.
36 | - Features delta migration to track and skip unchanged files in subsequent runs.
37 |
38 | ### Package `io.huangsam.photohaul.migration.state`
39 |
40 | - Manages state files for delta migration.
41 | - Tracks migrated files' paths, sizes, and timestamps.
42 | - Supports local and cloud-based state storage.
43 |
44 | ### Package `io.huangsam.photohaul.resolution`
45 |
46 | - Handles photo path resolution and organization.
47 | - Supports date-based folder structures for migrated photos.
48 | - Provides interfaces for custom resolution logic.
49 |
50 | ### Package `io.huangsam.photohaul.settings`
51 |
52 | - Manages configuration loading from properties files.
53 | - Handles API keys and migration settings.
54 | - Supports environment-specific configurations.
55 |
56 | ### `src/main/resources/`
57 |
58 | - Default config files: `config.properties`, provider-specific examples (e.g., `dropbox-example.properties`).
59 | - Static assets if needed.
60 |
61 | ### `src/test/java/io/huangsam/photohaul/`
62 |
63 | - Unit tests for all major classes.
64 | - Integration tests for end-to-end flows.
65 | - Uses JUnit 6, Mockito for mocking.
66 |
67 | ## Build Configuration (`build.gradle`)
68 |
69 | ### Plugins
70 |
71 | - `java`: Standard Java compilation.
72 | - `application`: Generates CLI scripts and handles main class execution.
73 | - `checkstyle`: Code style enforcement.
74 | - `jacoco`: Code coverage reporting (70% minimum).
75 | - `task-tree`: For visualizing task dependencies.
76 |
77 | ### Dependencies (via `gradle/libs.versions.toml`)
78 |
79 | - **Core**: JetBrains Annotations, SLF4J/Logback for logging.
80 | - **APIs**: Google Drive API, Dropbox SDK, SSHJ for SFTP, AWS SDK for S3.
81 | - **Metadata**: Drew Noakes' metadata-extractor for EXIF data.
82 | - **Testing**: JUnit 6, Mockito.
83 |
84 | ### Application Block
85 |
86 | - Main class: `io.huangsam.photohaul.Main`
87 | - JVM Args: `-Xmx1g` (1GB heap), `-XX:+UseG1GC` (G1 garbage collector) for performance tuning in photo processing.
88 |
89 | ### Tasks
90 |
91 | - `run`: Executes the app, forwards `-Dphotohaul.config` system property for custom configs.
92 | - `test`: Runs JUnit tests with Jacoco coverage.
93 | - `check`: Includes coverage verification.
94 | - `build`: Compiles, tests, and packages.
95 |
96 | ### Key Notes for Agents
97 |
98 | - **Performance Focus**: Recent optimizations include lazy metadata, multi-level deduplication, and delta migration.
99 | - **Thread Safety**: Photo metadata uses synchronized lazy loading; deduplication is stream-based and parallel-friendly.
100 | - **Config-Driven**: Behavior changes via properties files (e.g., migration type, API credentials).
101 | - **Error Handling**: Robust logging with SLF4J; exceptions in metadata/migration are caught and logged.
102 | - **Testing**: High coverage; mocks external APIs (Dropbox, Drive, S3) for reliable tests.
103 |
104 | For contributions or modifications, ensure tests pass and coverage stays above 70%.
105 |
106 | Use `./gradlew run -Dphotohaul.config=path/to/config.properties` for local testing.
107 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/migration/TestDropboxMigrator.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration;
2 |
3 | import com.dropbox.core.v2.DbxClientV2;
4 | import com.dropbox.core.v2.files.DbxUserFilesRequests;
5 | import com.dropbox.core.v2.files.ListFolderErrorException;
6 | import com.dropbox.core.v2.files.ListFolderResult;
7 | import com.dropbox.core.v2.files.UploadBuilder;
8 | import io.huangsam.photohaul.model.Photo;
9 | import io.huangsam.photohaul.resolution.PhotoResolver;
10 | import io.huangsam.photohaul.resolution.ResolutionException;
11 | import org.junit.jupiter.api.Test;
12 | import org.junit.jupiter.api.extension.ExtendWith;
13 | import org.mockito.Mock;
14 | import org.mockito.junit.jupiter.MockitoExtension;
15 |
16 | import static org.junit.jupiter.api.Assertions.assertEquals;
17 | import static org.junit.jupiter.api.Assertions.assertThrows;
18 | import static org.mockito.ArgumentMatchers.any;
19 | import static org.mockito.ArgumentMatchers.anyString;
20 | import static org.mockito.Mockito.times;
21 | import static org.mockito.Mockito.verify;
22 | import static org.mockito.Mockito.when;
23 |
24 | @ExtendWith(MockitoExtension.class)
25 | public class TestDropboxMigrator extends TestMigrationAbstract {
26 | @Mock
27 | DbxClientV2 clientMock;
28 |
29 | @Mock
30 | PhotoResolver photoResolverMock;
31 |
32 | @Mock
33 | DbxUserFilesRequests requestsMock;
34 |
35 | @Mock
36 | ListFolderResult folderResultMock;
37 |
38 | @Mock
39 | UploadBuilder uploadBuilderMock;
40 |
41 | @Test
42 | void testMigratePhotosNewFoldersSuccess() throws Exception {
43 | when(clientMock.files()).thenReturn(requestsMock);
44 | when(requestsMock.listFolder(anyString())).thenReturn(folderResultMock);
45 | when(requestsMock.uploadBuilder(anyString())).thenReturn(uploadBuilderMock);
46 |
47 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
48 |
49 | Migrator migrator = new DropboxMigrator("/Foobar", photoResolverMock, clientMock);
50 | run(migrator);
51 |
52 | verify(requestsMock, times(0)).createFolderV2(anyString());
53 | verify(requestsMock, times(2)).uploadBuilder(anyString());
54 |
55 | assertEquals(2, migrator.getSuccessCount());
56 | assertEquals(0, migrator.getFailureCount());
57 |
58 | migrator.close(); // No-op, but ensures no exception
59 | }
60 |
61 | @Test
62 | void testMigratePhotosOldFoldersSuccess() throws Exception {
63 | when(clientMock.files()).thenReturn(requestsMock);
64 | when(requestsMock.listFolder(anyString())).thenThrow(ListFolderErrorException.class);
65 | when(requestsMock.createFolderV2(anyString())).thenReturn(null);
66 | when(requestsMock.uploadBuilder(anyString())).thenReturn(uploadBuilderMock);
67 |
68 | when(photoResolverMock.resolveString(any(Photo.class))).thenReturn("some/path");
69 |
70 | Migrator migrator = new DropboxMigrator("/Foobar", photoResolverMock, clientMock);
71 | run(migrator);
72 |
73 | verify(requestsMock, times(2)).createFolderV2(anyString());
74 | verify(requestsMock, times(2)).uploadBuilder(anyString());
75 |
76 | assertEquals(2, migrator.getSuccessCount());
77 | assertEquals(0, migrator.getFailureCount());
78 |
79 | migrator.close(); // No-op, but ensures no exception
80 | }
81 |
82 | @Test
83 | @SuppressWarnings("resource")
84 | void testDropboxSetupNotWorking() {
85 | assertThrows(IllegalArgumentException.class, () ->
86 | new DropboxMigrator("NoSlashAtStart", photoResolverMock, clientMock));
87 | }
88 |
89 | @Test
90 | void testMigratePhotosWithResolutionException() throws Exception {
91 | when(clientMock.files()).thenReturn(requestsMock);
92 | when(requestsMock.listFolder(anyString())).thenReturn(folderResultMock);
93 | when(requestsMock.uploadBuilder(anyString())).thenReturn(uploadBuilderMock);
94 |
95 | when(photoResolverMock.resolveString(any(Photo.class))).thenThrow(new ResolutionException("Resolution failed"));
96 |
97 | Migrator migrator = new DropboxMigrator("/Foobar", photoResolverMock, clientMock);
98 | run(migrator);
99 |
100 | verify(requestsMock, times(0)).createFolderV2(anyString());
101 | verify(requestsMock, times(2)).uploadBuilder(anyString());
102 |
103 | assertEquals(2, migrator.getSuccessCount());
104 | assertEquals(0, migrator.getFailureCount());
105 |
106 | migrator.close(); // No-op, but ensures no exception
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/src/test/java/io/huangsam/photohaul/TestSettings.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul;
2 |
3 | import org.jspecify.annotations.NonNull;
4 | import org.junit.jupiter.api.Test;
5 | import org.junit.jupiter.api.io.TempDir;
6 |
7 | import java.io.IOException;
8 | import java.nio.file.Files;
9 | import java.nio.file.Path;
10 | import java.util.Properties;
11 |
12 | import static org.junit.jupiter.api.Assertions.assertEquals;
13 | import static org.junit.jupiter.api.Assertions.assertFalse;
14 | import static org.junit.jupiter.api.Assertions.assertNotNull;
15 | import static org.junit.jupiter.api.Assertions.assertThrows;
16 | import static org.junit.jupiter.api.Assertions.assertTrue;
17 |
18 | public class TestSettings {
19 | @Test
20 | void testGetSourcePath() {
21 | Settings settings = new Settings("path-example.properties");
22 | assertTrue(settings.getSourcePath().endsWith("Dummy/Source"));
23 | }
24 |
25 | @Test
26 | void testGetValueIsValid() {
27 | Settings settings = new Settings("dbx-example.properties");
28 | assertTrue(settings.getValue("dbx.target").startsWith("/"));
29 | }
30 |
31 | @Test
32 | void testGetValueIsMissing() {
33 | Settings settings = new Settings("drive-example.properties");
34 | assertThrows(NullPointerException.class, () -> settings.getValue("foo.bar"));
35 | }
36 |
37 | @Test
38 | void testGetValueWithFallback() {
39 | Settings settings = new Settings("path-example.properties");
40 | String expected = "foo";
41 | assertEquals(expected, settings.getValue("foo.bar", expected));
42 | }
43 |
44 | @Test
45 | void testSettingsFromProperties() {
46 | Properties properties = new Properties();
47 | properties.setProperty("hello.message", "world");
48 | Settings settings = new Settings(properties);
49 | assertEquals("world", settings.getValue("hello.message"));
50 | }
51 |
52 | @Test
53 | void testGetDefaultConfig() {
54 | assertNotNull(Settings.getDefault());
55 | }
56 |
57 | @Test
58 | void testGetMigratorMode() {
59 | Settings settings = new Settings("path-example.properties");
60 | assertEquals(io.huangsam.photohaul.migration.MigratorMode.PATH, settings.getMigratorMode());
61 | }
62 |
63 | @Test
64 | void testLoadFromFilesystem(@TempDir @NonNull Path tmp) throws IOException {
65 | // Arrange: create a temp properties file
66 | Path props = tmp.resolve("custom.properties");
67 | Files.writeString(props, "hello=filesystem\n");
68 |
69 | // Act: load using absolute filesystem path
70 | Settings settings = new Settings(props.toString());
71 |
72 | // Assert
73 | assertEquals("filesystem", settings.getValue("hello"));
74 | }
75 |
76 | @Test
77 | void testMissingSettingsThrows() {
78 | assertThrows(IllegalStateException.class, () -> new Settings("__definitely_not_here__.properties"));
79 | }
80 |
81 | @Test
82 | void testSystemPropertyOverrideFilesystem(@TempDir @NonNull Path tmp) throws IOException {
83 | Path props = tmp.resolve("override.properties");
84 | Files.writeString(props, "migrator.mode=PATH\nfoo.bar=baz\n");
85 | String original = System.getProperty("photohaul.config");
86 | try {
87 | System.setProperty("photohaul.config", props.toString());
88 | Settings settings = Settings.getDefault();
89 | assertEquals("baz", settings.getValue("foo.bar"));
90 | assertEquals(io.huangsam.photohaul.migration.MigratorMode.PATH, settings.getMigratorMode());
91 | } finally {
92 | if (original == null) {
93 | System.clearProperty("photohaul.config");
94 | } else {
95 | System.setProperty("photohaul.config", original);
96 | }
97 | }
98 | }
99 |
100 | @Test
101 | void testIsDeltaEnabledDefaultsFalse() {
102 | Settings settings = new Settings("path-example.properties");
103 | assertFalse(settings.isDeltaEnabled());
104 | }
105 |
106 | @Test
107 | void testIsDeltaEnabledWhenTrue(@TempDir @NonNull Path tmp) throws IOException {
108 | Path props = tmp.resolve("delta.properties");
109 | Files.writeString(props, "path.source=Dummy/Source\nmigrator.mode=PATH\ndelta.enabled=true\n");
110 | Settings settings = new Settings(props.toString());
111 | assertTrue(settings.isDeltaEnabled());
112 | }
113 |
114 | @Test
115 | void testIsDeltaEnabledWhenExplicitlyFalse(@TempDir @NonNull Path tmp) throws IOException {
116 | Path props = tmp.resolve("nodelta.properties");
117 | Files.writeString(props, "path.source=Dummy/Source\nmigrator.mode=PATH\ndelta.enabled=false\n");
118 | Settings settings = new Settings(props.toString());
119 | assertFalse(settings.isDeltaEnabled());
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/USERGUIDE.md:
--------------------------------------------------------------------------------
1 | # User guide
2 |
3 | Here's an extended version of content from the [general README](README.md).
4 |
5 | ## General setup
6 |
7 | If you have not built the application yet, please follow the general README
8 | to get started first.
9 |
10 | You can specify a different configuration file for various environments or specific scenarios by passing a JVM system property.
11 | Use the `-Dphotohaul.config=` flag when running the application. The value can be either the classpath resource name (relative to `src/main/resources`) or a filesystem path.
12 |
13 | Examples:
14 |
15 | ```shell
16 | # Use the default file (no override)
17 | ./gradlew run
18 |
19 | # Override with a classpath resource under src/main/resources/personal
20 | ./gradlew run -Dphotohaul.config=personal/path.properties
21 |
22 | # Or override with a filesystem path (absolute or relative)
23 | ./gradlew run -Dphotohaul.config=./src/main/resources/personal/path.properties
24 | ```
25 |
26 | Notes:
27 |
28 | - You can use either a classpath name (`personal/path.properties`) or a filesystem path.
29 | - Default is `config.properties` if not overridden.
30 |
31 | ### Path setup
32 |
33 | Configure the following property fields:
34 |
35 | - `migrator.mode`
36 | - `path.source`
37 | - `path.target`
38 | - `path.action`
39 |
40 | Refer to `PathMigrator` to learn more about the `path.action` values.
41 |
42 | ### Dropbox setup
43 |
44 | Configure the following property fields:
45 |
46 | - `migrator.mode`
47 | - `path.source`
48 | - `dbx.target`
49 | - `dbx.clientId`
50 | - `dbx.accessToken`
51 |
52 | [Click here](https://github.com/dropbox/dropbox-sdk-java?tab=readme-ov-file#dropbox-for-java-tutorial) to learn how to setup the `dbx` fields.
53 |
54 | ### Google Drive setup
55 |
56 | Configure the following property fields:
57 |
58 | - `migrator.mode`
59 | - `path.source`
60 | - `drive.target`
61 | - `drive.credentialFile`
62 | - `drive.appName`
63 |
64 | [Click here](https://developers.google.com/drive/api/quickstart/java#set-up-environment) to learn how to setup the `drive` fields.
65 |
66 | ### SFTP setup
67 |
68 | Configure the following property fields:
69 |
70 | - `migrator.mode`
71 | - `path.source`
72 | - `sftp.host`
73 | - `sftp.port` (optional, defaults to 22)
74 | - `sftp.username`
75 | - `sftp.password`
76 | - `sftp.target`
77 |
78 | SFTP (SSH File Transfer Protocol) is used for secure file transfers over SSH. Host keys are verified against your known hosts file for security. Ensure your server supports SFTP.
79 |
80 | ### S3 setup
81 |
82 | Configure the following property fields:
83 |
84 | - `migrator.mode`
85 | - `path.source`
86 | - `s3.bucket`
87 | - `s3.accessKey`
88 | - `s3.secretKey`
89 | - `s3.region` (optional, defaults to us-east-1)
90 |
91 | Amazon S3 (Simple Storage Service) is used for scalable cloud storage. You'll need AWS credentials with S3 permissions. [Click here](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html) to learn how to obtain and configure AWS credentials.
92 |
93 | ## Delta migration (optional)
94 |
95 | Delta migration is an optional feature that tracks which files have been migrated and skips unchanged files in subsequent runs. This significantly improves performance for large photo collections.
96 |
97 | ### How it works
98 |
99 | - Photohaul maintains a `.photohaul_state.json` file that records the path, size, and last modified timestamp of successfully migrated files
100 | - On subsequent runs, only new or modified files are migrated
101 | - For PATH mode, the state file is stored in the target directory
102 | - For cloud destinations (Dropbox, Drive, SFTP, S3), the state file is stored locally at the source path
103 |
104 | ### Enable delta migration
105 |
106 | Add the following property to your configuration file:
107 |
108 | ```properties
109 | delta.enabled=true
110 | ```
111 |
112 | Example for `PATH` mode:
113 |
114 | ```properties
115 | migrator.mode=PATH
116 | path.source=Dummy/Source
117 | path.target=Dummy/Target
118 | path.action=COPY
119 | delta.enabled=true
120 | ```
121 |
122 | **Note:** Delta migration is disabled by default to ensure backward compatibility.
123 |
124 | ## Run migration
125 |
126 | - Open your terminal and navigate to the `./photohaul` directory
127 | - Run the command `./gradlew run`. This will start the migration process
128 |
129 | ## Validate migration
130 |
131 | Once the migration is complete, you can verify that your photos are uploaded successfully.
132 |
133 | Below is an example of validating changes on Google Drive.
134 |
135 | ### Google Drive validation
136 |
137 | **Folder creation** was successful:
138 |
139 |
140 |
141 | Photo creation in **2015** was successful:
142 |
143 |
144 |
145 | Photo creation in **2024** was successful:
146 |
147 |
148 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/state/MigrationStateFile.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration.state;
2 |
3 | import com.google.gson.Gson;
4 | import com.google.gson.GsonBuilder;
5 | import com.google.gson.JsonSyntaxException;
6 | import com.google.gson.reflect.TypeToken;
7 | import org.jetbrains.annotations.NotNull;
8 | import org.jspecify.annotations.NonNull;
9 | import org.slf4j.Logger;
10 |
11 | import java.io.IOException;
12 | import java.lang.reflect.Type;
13 | import java.util.HashMap;
14 | import java.util.Map;
15 |
16 | import static org.slf4j.LoggerFactory.getLogger;
17 |
18 | /**
19 | * Manages the migration state file for delta migration.
20 | *
21 | *
The state file is a JSON file (e.g., .photohaul_state.json) stored at the
22 | * migration destination. It records the path, size, and last modified timestamp
23 | * of every successfully processed file, enabling efficient delta migrations by
24 | * skipping unchanged files.
25 | *
26 | *
This class provides an abstraction over the state file operations using
27 | * the {@link StateFileStorage} interface, allowing different storage backends
28 | * (local filesystem, S3, Dropbox, etc.) to be used.
29 | */
30 | public class MigrationStateFile {
31 | private static final Logger LOG = getLogger(MigrationStateFile.class);
32 | private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
33 | private static final Type STATE_MAP_TYPE = new TypeToken>() {}.getType();
34 |
35 | public static final String DEFAULT_STATE_FILE_NAME = ".photohaul_state.json";
36 |
37 | private final @NonNull StateFileStorage storage;
38 | private final @NonNull String stateFileName;
39 | private final @NonNull Map state;
40 |
41 | /**
42 | * Creates a MigrationStateFile with the default state file name.
43 | *
44 | * @param storage the storage backend for reading/writing state
45 | */
46 | public MigrationStateFile(@NotNull StateFileStorage storage) {
47 | this(storage, DEFAULT_STATE_FILE_NAME);
48 | }
49 |
50 | /**
51 | * Creates a MigrationStateFile with a custom state file name.
52 | *
53 | * @param storage the storage backend for reading/writing state
54 | * @param stateFileName the name of the state file
55 | */
56 | public MigrationStateFile(@NotNull StateFileStorage storage, @NotNull String stateFileName) {
57 | this.storage = storage;
58 | this.stateFileName = stateFileName;
59 | this.state = new HashMap<>();
60 | }
61 |
62 | /**
63 | * Load state from the storage backend.
64 | *
65 | *
This migrator wraps an existing migrator and uses a state file to track
23 | * previously migrated files. Before migration, it compares each file's current
24 | * metadata (size and last modified time) against the recorded state, and only
25 | * migrates files that are new or modified since the last run.
26 | *
27 | *
After each successful migration batch, the state file is updated with
28 | * the current state of all successfully migrated files.
29 | *
30 | *
This class identifies duplicate photos by computing their SHA-256 hash
25 | * and keeps only the first occurrence of each unique file.
26 | */
27 | public class PhotoDeduplicator {
28 | private static final Logger LOG = getLogger(PhotoDeduplicator.class);
29 | private static final String HASH_ALGORITHM = "SHA-256";
30 |
31 | /**
32 | * Deduplicate a collection of photos based on their SHA-256 hash.
33 | *
34 | *
For each photo, calculate its SHA-256 hash. If multiple photos have
35 | * the same hash, only the first occurrence is kept. The order of photos
36 | * in the input collection determines which photo is kept.
37 | *
38 | *
140 |
141 | Photo creation in **2015** was successful:
142 |
143 | 
144 |
145 | Photo creation in **2024** was successful:
146 |
147 | 
148 |
--------------------------------------------------------------------------------
/src/main/java/io/huangsam/photohaul/migration/state/MigrationStateFile.java:
--------------------------------------------------------------------------------
1 | package io.huangsam.photohaul.migration.state;
2 |
3 | import com.google.gson.Gson;
4 | import com.google.gson.GsonBuilder;
5 | import com.google.gson.JsonSyntaxException;
6 | import com.google.gson.reflect.TypeToken;
7 | import org.jetbrains.annotations.NotNull;
8 | import org.jspecify.annotations.NonNull;
9 | import org.slf4j.Logger;
10 |
11 | import java.io.IOException;
12 | import java.lang.reflect.Type;
13 | import java.util.HashMap;
14 | import java.util.Map;
15 |
16 | import static org.slf4j.LoggerFactory.getLogger;
17 |
18 | /**
19 | * Manages the migration state file for delta migration.
20 | *
21 | *