├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── src ├── test │ ├── resources │ │ ├── test │ │ │ ├── reader │ │ │ │ ├── artist.xml.gz │ │ │ │ ├── label.xml.gz │ │ │ │ ├── master.xml.gz │ │ │ │ └── release.xml.gz │ │ │ ├── ParseLongValueTestValidExample.xml │ │ │ ├── ParseLongValueTestMalformedExample.xml │ │ │ ├── IsXmlGZipEntryInvalidExample.xml │ │ │ ├── DiscogsDataMissingFieldsExample.xml │ │ │ ├── GetTypeTestMalformedEntryExample.xml │ │ │ ├── DiscogsDataValidExample.xml │ │ │ └── UTCLastModifiedMethodTestMalformedExample.xml │ │ ├── application-test-h2.yml │ │ ├── application-batch-test.yml │ │ └── application-test.yml │ └── java │ │ └── io │ │ └── dsub │ │ └── discogs │ │ └── batch │ │ ├── job │ │ ├── PostgreSQLDiscogsJobIntegrationTest.java │ │ ├── PostgreSQLIntegrationTestConfig.java │ │ ├── reader │ │ │ └── DumpItemReaderBuilderTest.java │ │ ├── BatchInfrastructureConfigTest.java │ │ ├── step │ │ │ └── AbstractStepConfigTest.java │ │ ├── tasklet │ │ │ └── FileClearTaskletTest.java │ │ └── listener │ │ │ └── ItemCountingItemProcessListenerTest.java │ │ ├── datasource │ │ └── DBTypeTest.java │ │ ├── argument │ │ ├── validator │ │ │ ├── KnownArgumentValidatorUnitTest.java │ │ │ ├── TypeArgumentValidatorTest.java │ │ │ ├── CompositeArgumentValidatorUnitTest.java │ │ │ ├── DataSourceArgumentValidatorUnitTest.java │ │ │ └── MappedValueValidatorUnitTest.java │ │ ├── handler │ │ │ └── DefaultArgumentHandlerIntegrationTest.java │ │ └── formatter │ │ │ └── JdbcUrlFormatterTest.java │ │ ├── service │ │ ├── DefaultDatabaseConnectionValidatorIntegrationTest.java │ │ └── DefaultDatabaseConnectionValidatorUnitTest.java │ │ ├── condition │ │ └── RequiresDiscogsDataConnection.java │ │ ├── container │ │ └── PostgreSQLContainerBaseTest.java │ │ ├── dump │ │ └── repository │ │ │ └── DiscogsDiscogsDumpRepositoryIntegrationTest.java │ │ ├── BatchApplicationTest.java │ │ ├── testutil │ │ └── LogSpy.java │ │ ├── util │ │ └── DiscogsJobParametersConverterUnitTest.java │ │ ├── BatchServiceTest.java │ │ └── JobLaunchingRunnerTest.java └── main │ ├── java │ └── io │ │ └── dsub │ │ └── discogs │ │ └── batch │ │ ├── domain │ │ ├── BaseXML.java │ │ ├── SubItemXML.java │ │ ├── master │ │ │ ├── MasterMainReleaseXML.java │ │ │ ├── MasterXML.java │ │ │ └── MasterSubItemsXML.java │ │ ├── HashXML.java │ │ ├── artist │ │ │ ├── ArtistXML.java │ │ │ └── ArtistSubItemsXML.java │ │ ├── label │ │ │ ├── LabelXML.java │ │ │ └── LabelSubItemsXML.java │ │ └── release │ │ │ └── ReleaseItemXML.java │ │ ├── exception │ │ ├── BaseRuntimeException.java │ │ ├── DumpNotFoundException.java │ │ ├── FileDeleteException.java │ │ ├── InvalidArgumentException.java │ │ ├── DriverLoadFailureException.java │ │ ├── InitializationFailureException.java │ │ ├── MissingRequiredParamsException.java │ │ ├── MissingRequiredArgumentException.java │ │ ├── FileException.java │ │ └── BaseCheckedException.java │ │ ├── datasource │ │ ├── DataSourceDetails.java │ │ └── DBType.java │ │ ├── dump │ │ ├── DumpSupplier.java │ │ ├── DumpDependencyResolver.java │ │ ├── service │ │ │ └── DiscogsDumpService.java │ │ ├── EntityType.java │ │ ├── repository │ │ │ └── DiscogsDumpRepository.java │ │ └── DiscogsDump.java │ │ ├── util │ │ ├── MalformedDateParser.java │ │ ├── ToggleProgressBarConsumer.java │ │ ├── ProgressBarUtil.java │ │ ├── FileUtil.java │ │ ├── DataSourceUtil.java │ │ └── DefaultMalformedDateParser.java │ │ ├── job │ │ ├── writer │ │ │ ├── JooqItemWriter.java │ │ │ ├── DefaultJooqMasterMainReleaseItemWriter.java │ │ │ ├── CollectionItemWriter.java │ │ │ ├── ItemWriterConfig.java │ │ │ └── DefaultLJooqItemWriter.java │ │ ├── JobParameterResolver.java │ │ ├── listener │ │ │ ├── StringNormalizingItemReadListener.java │ │ │ ├── ExitSignalJobExecutionListener.java │ │ │ ├── ItemCountingItemProcessListener.java │ │ │ ├── ClearanceJobExecutionListener.java │ │ │ ├── BatchListenerConfig.java │ │ │ ├── CacheInversionStepExecutionListener.java │ │ │ ├── StopWatchStepExecutionListener.java │ │ │ └── IdCachingItemProcessListener.java │ │ ├── step │ │ │ ├── GlobalStepConfig.java │ │ │ └── AbstractStepConfig.java │ │ ├── processor │ │ │ ├── ArtistCoreProcessor.java │ │ │ ├── LabelCoreProcessor.java │ │ │ ├── MasterCoreProcessor.java │ │ │ ├── MasterMainReleaseItemProcessor.java │ │ │ ├── LabelSubItemsProcessor.java │ │ │ ├── ReleaseItemCoreProcessor.java │ │ │ └── ItemProcessorConfig.java │ │ ├── registry │ │ │ ├── EntityIdRegistry.java │ │ │ ├── DefaultEntityIdRegistry.java │ │ │ └── IdCache.java │ │ ├── UniqueRunIdIncrementer.java │ │ ├── reader │ │ │ └── DiscogsDumpItemReaderBuilder.java │ │ ├── tasklet │ │ │ ├── FileClearTasklet.java │ │ │ └── GenreStyleInsertionTasklet.java │ │ ├── decider │ │ │ └── MasterMainReleaseStepJobExecutionDecider.java │ │ ├── DefaultJobParameterResolver.java │ │ └── BatchInfrastructureConfig.java │ │ ├── argument │ │ ├── formatter │ │ │ ├── ArgumentFormatter.java │ │ │ ├── CompositeArgumentFormatter.java │ │ │ ├── FlagRemovingArgumentFormatter.java │ │ │ ├── ArgumentNameFormatter.java │ │ │ └── JdbcUrlFormatter.java │ │ ├── validator │ │ │ ├── ArgumentValidator.java │ │ │ ├── DatabaseConnectionValidator.java │ │ │ ├── KnownArgumentValidator.java │ │ │ ├── ValidationResult.java │ │ │ ├── TypeArgumentValidator.java │ │ │ └── CompositeArgumentValidator.java │ │ ├── handler │ │ │ └── ArgumentHandler.java │ │ └── ArgType.java │ │ ├── LiquibaseConfig.java │ │ ├── config │ │ ├── JooqConfig.java │ │ ├── DiscogsBatchConfigurer.java │ │ ├── BatchConfig.java │ │ └── TaskExecutorConfig.java │ │ ├── BatchApplication.java │ │ ├── BatchService.java │ │ ├── JobLaunchingRunner.java │ │ └── JobPreparationRunner.java │ └── resources │ └── application.yml ├── .travis.yml ├── .github └── workflows │ └── publish.yml └── gradlew.bat /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'discogs-batch' 2 | 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/echovisionlab/discogs-batch/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/test/resources/test/reader/artist.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/echovisionlab/discogs-batch/HEAD/src/test/resources/test/reader/artist.xml.gz -------------------------------------------------------------------------------- /src/test/resources/test/reader/label.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/echovisionlab/discogs-batch/HEAD/src/test/resources/test/reader/label.xml.gz -------------------------------------------------------------------------------- /src/test/resources/test/reader/master.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/echovisionlab/discogs-batch/HEAD/src/test/resources/test/reader/master.xml.gz -------------------------------------------------------------------------------- /src/test/resources/test/reader/release.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/echovisionlab/discogs-batch/HEAD/src/test/resources/test/reader/release.xml.gz -------------------------------------------------------------------------------- /src/test/resources/test/ParseLongValueTestValidExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 33 3 | 777 4 | 3323238 5 | 3321 6 | -------------------------------------------------------------------------------- /src/test/resources/test/ParseLongValueTestMalformedExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | d 4 | 3323d 5 | 33211d22!!# 6 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/BaseXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain; 2 | 3 | import org.jooq.UpdatableRecord; 4 | 5 | public interface BaseXML> { 6 | 7 | T buildRecord(); 8 | } 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk16 4 | before_script: 5 | - chmod +x gradlew 6 | script: 7 | - ./gradlew check 8 | - ./gradlew jacocoTestReport 9 | after_success: 10 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dist -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/SubItemXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain; 2 | 3 | import org.jooq.UpdatableRecord; 4 | 5 | public interface SubItemXML> { 6 | 7 | T getRecord(int parentId); 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/BaseRuntimeException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public abstract class BaseRuntimeException extends RuntimeException { 4 | public BaseRuntimeException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/DumpNotFoundException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class DumpNotFoundException extends BaseRuntimeException { 4 | 5 | public DumpNotFoundException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/FileDeleteException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class FileDeleteException extends FileException { 4 | public FileDeleteException(String message, Throwable cause) { 5 | super(message, cause); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/InvalidArgumentException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class InvalidArgumentException extends BaseRuntimeException { 4 | 5 | public InvalidArgumentException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/DriverLoadFailureException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class DriverLoadFailureException extends BaseCheckedException { 4 | 5 | public DriverLoadFailureException(String message) { 6 | super(message); 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/InitializationFailureException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class InitializationFailureException extends BaseRuntimeException { 4 | 5 | public InitializationFailureException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/MissingRequiredParamsException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class MissingRequiredParamsException extends BaseRuntimeException { 4 | 5 | public MissingRequiredParamsException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/MissingRequiredArgumentException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class MissingRequiredArgumentException extends BaseRuntimeException { 4 | 5 | public MissingRequiredArgumentException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/datasource/DataSourceDetails.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.datasource; 2 | 3 | import javax.sql.DataSource; 4 | import org.jooq.SQLDialect; 5 | 6 | /** 7 | * An immutable wrapper for datasource details. 8 | */ 9 | public record DataSourceDetails(DataSource dataSource, SQLDialect dialect, DBType type) { 10 | 11 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/DumpSupplier.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump; 2 | 3 | import java.io.File; 4 | import java.util.List; 5 | import java.util.function.Supplier; 6 | 7 | public interface DumpSupplier extends Supplier> { 8 | 9 | List get(); 10 | 11 | List get(File file); 12 | } 13 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/PostgreSQLDiscogsJobIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import org.springframework.test.context.ContextConfiguration; 4 | 5 | @ContextConfiguration(classes = PostgreSQLIntegrationTestConfig.class) 6 | public class PostgreSQLDiscogsJobIntegrationTest extends DiscogsJobIntegrationTest { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/FileException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class FileException extends BaseCheckedException { 4 | 5 | public FileException(String message) { 6 | super(message); 7 | } 8 | 9 | public FileException(String message, Throwable cause) { 10 | super(message, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/MalformedDateParser.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import java.time.LocalDate; 4 | 5 | public interface MalformedDateParser { 6 | 7 | boolean isMonthValid(String date); 8 | 9 | boolean isYearValid(String date); 10 | 11 | boolean isDayValid(String date); 12 | 13 | LocalDate parse(String date); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/exception/BaseCheckedException.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.exception; 2 | 3 | public class BaseCheckedException extends Exception { 4 | 5 | public BaseCheckedException(String message) { 6 | super(message); 7 | } 8 | 9 | public BaseCheckedException(String message, Throwable cause) { 10 | super(message, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/test/resources/application-test-h2.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | datasource: 3 | username: sa 4 | password: 5 | url: jdbc:h2:mem:testdb;MODE=MYSQL 6 | driver-class-name: org.h2.Driver 7 | tomcat: 8 | test-while-idle: true 9 | validation-query: SELECT 1 10 | jpa: 11 | hibernate: 12 | ddl-auto: update 13 | generate-ddl: true 14 | database-platform: org.hibernate.dialect.H2Dialect 15 | main: 16 | banner-mode: OFF -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/writer/JooqItemWriter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.writer; 2 | 3 | import java.util.List; 4 | import org.jooq.Query; 5 | import org.jooq.UpdatableRecord; 6 | import org.springframework.batch.item.ItemWriter; 7 | 8 | public interface JooqItemWriter> extends ItemWriter { 9 | 10 | @Override 11 | void write(List items); 12 | 13 | Query getQuery(T record); 14 | } 15 | -------------------------------------------------------------------------------- /src/test/resources/application-batch-test.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | batch: 3 | job: 4 | enabled: false 5 | data: 6 | jpa: 7 | repositories: 8 | # this is mandate as of initialization of JPA repositories 9 | # may live under the process EVEN AFTER the batch job is finished. 10 | # this is also can be a potential reason of spring batch not being exited 11 | # immediately after the job is done. 12 | bootstrap-mode: default -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/JobParameterResolver.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import io.dsub.discogs.batch.exception.DumpNotFoundException; 4 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 5 | import java.util.Properties; 6 | import org.springframework.boot.ApplicationArguments; 7 | 8 | public interface JobParameterResolver { 9 | 10 | Properties resolve(ApplicationArguments applicationArguments) 11 | throws InvalidArgumentException, DumpNotFoundException; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/DumpDependencyResolver.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump; 2 | 3 | import io.dsub.discogs.batch.exception.DumpNotFoundException; 4 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 5 | import java.util.Collection; 6 | import org.springframework.boot.ApplicationArguments; 7 | 8 | public interface DumpDependencyResolver { 9 | 10 | Collection resolve(ApplicationArguments args) 11 | throws DumpNotFoundException, InvalidArgumentException; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/formatter/ArgumentFormatter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | /** 4 | * Argument formatter interface to present formatter of specific argument value. 5 | */ 6 | public interface ArgumentFormatter { 7 | 8 | /** 9 | * Single method that performs formatting. 10 | * 11 | * @param args arguments to be evaluated. 12 | * @return result that is either being formatted, or ignored to be formatted. 13 | */ 14 | String[] format(String[] args); 15 | } 16 | -------------------------------------------------------------------------------- /src/test/resources/application-test.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | datasource: 3 | hikari: 4 | data-source-properties: 5 | rewriteBatchedStatements: true # JDBC BATCH 6 | batch: 7 | job: 8 | enabled: false 9 | jpa: 10 | properties: 11 | hibernate: 12 | jdbc: 13 | # JPA_HIBERNATE_BATCH_PROPS 14 | batch_size: 30 # GREATER THAN 0 15 | order_inserts: true # IMPROVES RELATIONAL INSERT 16 | order_updates: true # IMPROVES UPDATE RELATIONAL ENTITIES 17 | main: 18 | banner-mode: OFF 19 | -------------------------------------------------------------------------------- /src/test/resources/test/IsXmlGZipEntryInvalidExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | "8fbe2987c0c4298b4fce157930495bb9-3" 4 | 2017-02-16T15:05:11.000Z 5 | 22233075 6 | STANDARD 7 | 8 | 9 | "00c63a4c5cb686c350b78a79b1ca3800" 10 | 11 | 2017-02-16T15:05:15.000Z 12 | 4500237 13 | STANDARD 14 | 15 | -------------------------------------------------------------------------------- /src/test/resources/test/DiscogsDataMissingFieldsExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | "8fbe2987c0c4298b4fce157930495bb9-3" 4 | data/2008/discogs_20080309_artists.xml.gz 5 | 2017-02-16T15:05:11.000Z 6 | 22233075 7 | STANDARD 8 | 9 | 10 | 11 | data/2008/discogs_20080309_labels.xml.gz 12 | 2017-02-16T15:05:15.000Z 13 | 14 | STANDARD 15 | 16 | -------------------------------------------------------------------------------- /src/test/resources/test/GetTypeTestMalformedEntryExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | "8fbe2987c0c4298b4fce157930495bb9-3" 4 | data/2008/discogs_20080309 5 | 2017-02-16T15:05:11.000Z 6 | 22233075 7 | STANDARD 8 | 9 | 10 | "00c63a4c5cb686c350b78a79b1ca3800" 11 | data/2008/discogs_20080309 12 | 2017-02-16T15:05:15.000Z 13 | 4500237 14 | STANDARD 15 | 16 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/ArgumentValidator.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import org.springframework.boot.ApplicationArguments; 4 | 5 | /** 6 | * an interface representing a validator for {@link ApplicationArguments}. actual validation process 7 | * are dependent on its implementation. 8 | */ 9 | public interface ArgumentValidator { 10 | 11 | /** 12 | * method to validate {@link ApplicationArguments}. 13 | * 14 | * @param applicationArguments to be validated. 15 | * @return result of validation represented as {@link ValidationResult}. 16 | */ 17 | ValidationResult validate(ApplicationArguments applicationArguments); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/StringNormalizingItemReadListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import io.dsub.discogs.batch.util.ReflectionUtil; 4 | import org.springframework.batch.core.ItemReadListener; 5 | import org.springframework.lang.NonNull; 6 | 7 | public class StringNormalizingItemReadListener implements ItemReadListener { 8 | 9 | /* No Op */ 10 | @Override 11 | public void beforeRead() { 12 | } 13 | 14 | @Override 15 | public void afterRead(@NonNull Object item) { 16 | ReflectionUtil.normalizeStringFields(item); 17 | } 18 | 19 | /* No Op */ 20 | @Override 21 | public void onReadError(@NonNull Exception ex) { 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/test/resources/test/DiscogsDataValidExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | "8fbe2987c0c4298b4fce157930495bb9-3" 4 | data/2008/discogs_20080309_artists.xml.gz 5 | 2017-02-16T15:05:11.000Z 6 | 22233075 7 | STANDARD 8 | 9 | 10 | "00c63a4c5cb686c350b78a79b1ca3800" 11 | data/2008/discogs_20080309_labels.xml.gz 12 | 2017-02-16T15:05:15.000Z 13 | 4500237 14 | STANDARD 15 | 16 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to GitHub Packages 2 | on: 3 | release: 4 | types: [created] # creating release from repo will auto trigger this action! 5 | jobs: 6 | publish: 7 | runs-on: ubuntu-latest 8 | permissions: 9 | contents: read 10 | packages: write 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-java@v2 14 | with: 15 | java-version: '16' 16 | distribution: 'adopt' 17 | - name: Publish package 18 | run: gradle publish 19 | env: 20 | # token that has access to package-related actions 21 | PKG_TOKEN: ${{ secrets.PKG_TOKEN }} 22 | # github username 23 | USERNAME: ${{ secrets.USERNAME }} -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/step/GlobalStepConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.step; 2 | 3 | import io.dsub.discogs.batch.job.step.core.ArtistStepConfig; 4 | import io.dsub.discogs.batch.job.step.core.LabelStepConfig; 5 | import io.dsub.discogs.batch.job.step.core.MasterStepConfig; 6 | import io.dsub.discogs.batch.job.step.core.ReleaseItemStepConfig; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.context.annotation.Import; 9 | 10 | @Configuration 11 | @Import( 12 | value = { 13 | ArtistStepConfig.class, 14 | LabelStepConfig.class, 15 | MasterStepConfig.class, 16 | ReleaseItemStepConfig.class 17 | }) 18 | public class GlobalStepConfig { 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/handler/ArgumentHandler.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.handler; 2 | 3 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 4 | 5 | /** 6 | * argument handler that takes care about the application argument. i.e. jdbc url, username, 7 | * password existence, formatting, etc. 8 | */ 9 | public interface ArgumentHandler { 10 | 11 | /** 12 | * single method to resolve if argument requirements are met. 13 | * 14 | * @param args given arguments. 15 | * @return resolved arguments, or corrected arguments. 16 | * @throws InvalidArgumentException thrown if argument requirements are not met. 17 | */ 18 | String[] resolve(String[] args) throws InvalidArgumentException; 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/ArtistCoreProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.artist.ArtistXML; 4 | import io.dsub.discogs.batch.util.ReflectionUtil; 5 | import io.dsub.discogs.jooq.tables.records.ArtistRecord; 6 | import lombok.RequiredArgsConstructor; 7 | import org.springframework.batch.item.ItemProcessor; 8 | 9 | @RequiredArgsConstructor 10 | public class ArtistCoreProcessor implements ItemProcessor { 11 | 12 | @Override 13 | public ArtistRecord process(ArtistXML item) { 14 | if (item.getId() == null || item.getId() < 1) { 15 | return null; 16 | } 17 | ReflectionUtil.normalizeStringFields(item); 18 | return item.buildRecord(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/LiquibaseConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import liquibase.integration.spring.SpringLiquibase; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | import javax.sql.DataSource; 9 | 10 | /** 11 | * Liquibase Changelog 12 | **/ 13 | @Slf4j 14 | @Configuration 15 | public class LiquibaseConfig { 16 | @Bean 17 | public SpringLiquibase liquibase(DataSource dataSource) { 18 | SpringLiquibase liquibase = new SpringLiquibase(); 19 | liquibase.setChangeLog("db/changelog/db.changelog-master.yaml"); 20 | liquibase.setShouldRun(true); 21 | liquibase.setDataSource(dataSource); 22 | return liquibase; 23 | } 24 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/LabelCoreProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.label.LabelXML; 4 | import io.dsub.discogs.batch.util.ReflectionUtil; 5 | import io.dsub.discogs.jooq.tables.records.LabelRecord; 6 | import lombok.RequiredArgsConstructor; 7 | import org.springframework.batch.item.ItemProcessor; 8 | 9 | @RequiredArgsConstructor 10 | public class LabelCoreProcessor implements ItemProcessor { 11 | 12 | @Override 13 | public LabelRecord process(LabelXML command) throws Exception { 14 | if (command.getId() == null || command.getId() < 1) { 15 | return null; 16 | } 17 | ReflectionUtil.normalizeStringFields(command); 18 | return command.buildRecord(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/MasterCoreProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.master.MasterXML; 4 | import io.dsub.discogs.batch.util.ReflectionUtil; 5 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 6 | import lombok.RequiredArgsConstructor; 7 | import org.springframework.batch.item.ItemProcessor; 8 | 9 | @RequiredArgsConstructor 10 | public class MasterCoreProcessor implements ItemProcessor { 11 | 12 | @Override 13 | public MasterRecord process(MasterXML master) throws Exception { 14 | if (master.getId() == null || master.getId() < 1) { 15 | return null; 16 | } 17 | ReflectionUtil.normalizeStringFields(master); 18 | return master.buildRecord(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/registry/EntityIdRegistry.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.registry; 2 | 3 | import java.util.concurrent.ConcurrentSkipListSet; 4 | 5 | /** 6 | * Entity ID cache to reduce DB lookups for entry. 7 | * The behaviors are up to its implementation. 8 | */ 9 | public interface EntityIdRegistry { 10 | 11 | boolean exists(Type type, Integer id); 12 | 13 | boolean exists(Type type, String id); 14 | 15 | void put(Type type, Integer id); 16 | 17 | void put(Type type, String id); 18 | 19 | void invert(Type type); 20 | 21 | void clearAll(); 22 | 23 | ConcurrentSkipListSet getStringIdSetByType(Type type); 24 | 25 | IdCache getLongIdCache(Type type); 26 | 27 | enum Type { 28 | ARTIST, LABEL, MASTER, RELEASE, GENRE, STYLE 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/ExitSignalJobExecutionListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import java.util.concurrent.CountDownLatch; 4 | import lombok.RequiredArgsConstructor; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.batch.core.JobExecution; 7 | import org.springframework.batch.core.JobExecutionListener; 8 | 9 | @Slf4j 10 | @RequiredArgsConstructor 11 | public class ExitSignalJobExecutionListener implements JobExecutionListener { 12 | 13 | private final CountDownLatch exitLatch; 14 | 15 | /* no op */ 16 | @Override 17 | public void beforeJob(JobExecution jobExecution) { 18 | } 19 | 20 | @Override 21 | public void afterJob(JobExecution jobExecution) { 22 | log.info("signal exit..."); 23 | exitLatch.countDown(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/PostgreSQLIntegrationTestConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import io.dsub.discogs.batch.container.PostgreSQLContainerBaseTest; 4 | import javax.sql.DataSource; 5 | import org.springframework.boot.ApplicationArguments; 6 | import org.springframework.boot.DefaultApplicationArguments; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | @Configuration 11 | public class PostgreSQLIntegrationTestConfig extends PostgreSQLContainerBaseTest { 12 | 13 | @Bean 14 | public DataSource dataSource() { 15 | return dataSource; 16 | } 17 | 18 | @Bean 19 | public ApplicationArguments applicationArguments() { 20 | return new DefaultApplicationArguments(jdbcUrl, username, password); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/UniqueRunIdIncrementer.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import org.springframework.batch.core.JobParameters; 4 | import org.springframework.batch.core.JobParametersBuilder; 5 | import org.springframework.batch.core.launch.support.RunIdIncrementer; 6 | 7 | public class UniqueRunIdIncrementer extends RunIdIncrementer { 8 | private static final String RUN_ID = "run.id"; 9 | 10 | @Override 11 | public JobParameters getNext(JobParameters parameters) { 12 | JobParameters params = (parameters == null) ? new JobParameters() : parameters; 13 | Long lastVal = params.getLong(RUN_ID, Long.valueOf(1)); 14 | lastVal = lastVal == null ? 1L : lastVal; 15 | return new JobParametersBuilder() 16 | .addLong(RUN_ID, lastVal + 1) 17 | .toJobParameters(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/resources/test/UTCLastModifiedMethodTestMalformedExample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | "8fbe2987c0c4298b4fce157930495bb9-3" 4 | data/2008/discogs_20080309_artists.xml.gz 5 | 11.000Z 6 | 22233075 7 | STANDARD 8 | 9 | 10 | "00c63a4c5cb686c350b78a79b1ca3800" 11 | data/2008/discogs_20080309_labels.xml.gz 12 | 2017-02-10Z 13 | 4500237 14 | STANDARD 15 | 16 | 17 | "00c63a4c5cb686c350b78a79b1ca3800" 18 | data/2008/discogs_20080309_labels.xml.gz 19 | 20 | 4500237 21 | STANDARD 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/datasource/DBTypeTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.datasource; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import java.util.List; 6 | import org.junit.jupiter.api.Test; 7 | import org.junit.jupiter.params.ParameterizedTest; 8 | import org.junit.jupiter.params.provider.EnumSource; 9 | 10 | class DBTypeTest { 11 | 12 | @EnumSource(value = DBType.class) 13 | @ParameterizedTest 14 | void whenGetDriverClassName__ShouldNotReturnNullOrBlank(DBType dbType) { 15 | // when 16 | String driverClassName = dbType.getDriverClassName(); 17 | 18 | // then 19 | assertThat(driverClassName) 20 | .isNotNull() 21 | .isNotBlank(); 22 | } 23 | 24 | @Test 25 | void whenGetNames__ShouldReturnLowerCases() { 26 | // when 27 | List names = DBType.getNames(); 28 | 29 | // then 30 | names.forEach(name -> assertThat(name).isNotBlank().isLowerCase()); 31 | } 32 | } -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | main: 3 | web-application-type: NONE 4 | banner-mode: OFF 5 | datasource: 6 | hikari: 7 | data-source-properties: 8 | rewriteBatchedStatements: true 9 | url: ${url} 10 | username: ${username} 11 | password: ${password} 12 | batch: 13 | job: 14 | enabled: false 15 | jpa: 16 | properties: 17 | hibernate: 18 | jdbc: 19 | batch_size: 500 20 | time_zone: UTC 21 | batch_versioned_data: true 22 | order_updates: true 23 | order_inserts: true 24 | format_sql: true 25 | hibernate: 26 | ddl-auto: update 27 | jmx: 28 | enabled: false 29 | logging: 30 | file: 31 | path: ./log 32 | level: 33 | liquibase: ERROR 34 | io.dsub: INFO 35 | org: 36 | reflections: ERROR 37 | springframework: ERROR 38 | hibernate: ERROR 39 | com.zaxxer.hikari: ERROR 40 | server: 41 | shutdown: graceful -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/formatter/CompositeArgumentFormatter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * Collection of argument formatter that delegates the format() method to its delegates. 8 | */ 9 | public class CompositeArgumentFormatter implements ArgumentFormatter { 10 | 11 | private final List delegates; 12 | 13 | public CompositeArgumentFormatter() { 14 | this.delegates = new ArrayList<>(); 15 | } 16 | 17 | public CompositeArgumentFormatter addFormatter(ArgumentFormatter additionalFormatter) { 18 | this.delegates.add(additionalFormatter); 19 | return this; 20 | } 21 | 22 | @Override 23 | public String[] format(String[] args) { 24 | String[] formatted = args; 25 | for (ArgumentFormatter delegate : delegates) { 26 | formatted = delegate.format(formatted); 27 | } 28 | return formatted; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/formatter/FlagRemovingArgumentFormatter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | import java.util.Arrays; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Removes all - or -- flags from arguments 9 | */ 10 | public class FlagRemovingArgumentFormatter implements ArgumentFormatter { 11 | 12 | private static final Pattern PATTERN = Pattern.compile("[-]*(.*)"); 13 | 14 | @Override 15 | public String[] format(String[] args) { 16 | if (args == null) { 17 | return null; 18 | } 19 | 20 | return Arrays.stream(args) 21 | .map(this::doFormat) 22 | .toArray(String[]::new); 23 | } 24 | 25 | private String doFormat(String arg) { 26 | if (arg == null || arg.isBlank()) { 27 | return arg; 28 | } 29 | Matcher m = PATTERN.matcher(arg); 30 | if (m.matches()) { 31 | return m.group(1); 32 | } 33 | return arg; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/DatabaseConnectionValidator.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import org.springframework.boot.ApplicationArguments; 4 | 5 | public interface DatabaseConnectionValidator extends ArgumentValidator { 6 | 7 | /** 8 | * validates database from url, username and password. 9 | * 10 | * @param url a jdbc url 11 | * @param username a username 12 | * @param password a password 13 | * @return blank result if no issues found, otherwise will contain one or many issues. 14 | */ 15 | ValidationResult validate(String url, String username, String password); 16 | 17 | /** 18 | * validates database from url, username and password. 19 | * 20 | * @param args a {@link ApplicationArguments} that contains url, username, password as option 21 | * argument. 22 | * @return blank result if no issues found, otherwise will contain one or many issues. 23 | */ 24 | ValidationResult validate(ApplicationArguments args); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/datasource/DBType.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.datasource; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | import lombok.Getter; 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.ToString; 9 | 10 | @ToString 11 | @RequiredArgsConstructor 12 | public enum DBType { 13 | POSTGRESQL("org.postgresql.Driver"); 14 | 15 | @Getter 16 | private final String driverClassName; 17 | 18 | public static List getNames() { 19 | return Arrays.stream(values()) 20 | .map(DBType::name) 21 | .map(String::toLowerCase) 22 | .collect(Collectors.toList()); 23 | } 24 | 25 | public static DBType getTypeOf(String from) { 26 | String target = from.toLowerCase(); 27 | return Arrays.stream(values()) 28 | .filter(type -> type.value().equals(target)) 29 | .findFirst() 30 | .orElse(null); 31 | } 32 | 33 | public String value() { 34 | return this.name().toLowerCase(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/service/DiscogsDumpService.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump.service; 2 | 3 | import io.dsub.discogs.batch.dump.DiscogsDump; 4 | import io.dsub.discogs.batch.dump.EntityType; 5 | import io.dsub.discogs.batch.exception.DumpNotFoundException; 6 | import java.util.Collection; 7 | import java.util.List; 8 | 9 | public interface DiscogsDumpService { 10 | 11 | void updateDB(); 12 | 13 | boolean exists(String eTag); 14 | 15 | DiscogsDump getDiscogsDump(String eTag) throws DumpNotFoundException; 16 | 17 | DiscogsDump getMostRecentDiscogsDumpByType(EntityType type); 18 | 19 | DiscogsDump getMostRecentDiscogsDumpByTypeYearMonth(EntityType type, int year, int month); 20 | 21 | Collection getAllByTypeYearMonth(List types, int year, int month) 22 | throws DumpNotFoundException; 23 | 24 | List getDumpByTypeInRange(EntityType type, int year, int month); 25 | 26 | List getLatestCompleteDumpSet() throws DumpNotFoundException; 27 | 28 | List getAll(); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/EntityType.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump; 2 | 3 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 4 | import java.util.List; 5 | import java.util.Locale; 6 | 7 | public enum EntityType { 8 | ARTIST, 9 | LABEL, 10 | MASTER, 11 | RELEASE; 12 | 13 | public static EntityType of(String name) throws InvalidArgumentException { 14 | String targetName = name.toLowerCase(Locale.US); 15 | for (EntityType value : values()) { 16 | if (value.toString().equals(targetName)) { 17 | return value; 18 | } 19 | } 20 | throw new InvalidArgumentException("failed to figure out type: " + name); 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return this.name().toLowerCase(Locale.US); 26 | } 27 | 28 | public List getDependencies() { 29 | if (this.equals(RELEASE)) { 30 | return List.of(values()); 31 | } 32 | if (this.equals(MASTER)) { 33 | return List.of(ARTIST, LABEL, MASTER); 34 | } 35 | return List.of(this); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/config/JooqConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.config; 2 | 3 | import io.dsub.discogs.batch.datasource.DataSourceDetails; 4 | import io.dsub.discogs.batch.util.DataSourceUtil; 5 | import javax.sql.DataSource; 6 | import lombok.RequiredArgsConstructor; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.jooq.DSLContext; 9 | import org.jooq.impl.DSL; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | 13 | /** 14 | * Infrastructure support for JOOQ configuration, providing the {@link DSLContext} bean. 15 | */ 16 | @Slf4j 17 | @Configuration 18 | @RequiredArgsConstructor 19 | public class JooqConfig { 20 | 21 | private final DataSource dataSource; 22 | 23 | @Bean 24 | public DSLContext dslContext() { 25 | DataSourceDetails details = dataSourceDetails(); 26 | return DSL.using(dataSource, details.dialect()); 27 | } 28 | 29 | @Bean 30 | public DataSourceDetails dataSourceDetails() { 31 | return DataSourceUtil.getDataSourceDetails(dataSource); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/BatchApplication.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; 6 | import org.springframework.context.ConfigurableApplicationContext; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | @Slf4j 10 | @SpringBootApplication(exclude = {HibernateJpaAutoConfiguration.class}) 11 | public class BatchApplication { 12 | 13 | private static ConfigurableApplicationContext APP_CONTEXT; 14 | 15 | public static void main(String[] args) { 16 | BatchService service = getBatchService(); 17 | try { 18 | APP_CONTEXT = service.run(args); 19 | } catch (Exception e) { 20 | log.error(e.getMessage(), e); 21 | } 22 | } 23 | 24 | protected static BatchService getBatchService() { 25 | return new BatchService(); 26 | } 27 | 28 | @Bean(name = "parentContext") 29 | public ConfigurableApplicationContext parentContext() { 30 | return APP_CONTEXT; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/MasterMainReleaseItemProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.master.MasterMainReleaseXML; 4 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 5 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 6 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 7 | import lombok.RequiredArgsConstructor; 8 | import org.springframework.batch.item.ItemProcessor; 9 | 10 | @RequiredArgsConstructor 11 | public class MasterMainReleaseItemProcessor 12 | implements ItemProcessor { 13 | 14 | private final EntityIdRegistry idRegistry; 15 | 16 | @Override 17 | public MasterRecord process(MasterMainReleaseXML item) throws Exception { 18 | if (item == null || item.getId() == null || item.getMainReleaseId() == null) { 19 | return null; 20 | } 21 | 22 | if (!idRegistry.exists(DefaultEntityIdRegistry.Type.RELEASE, item.getMainReleaseId())) { 23 | return null; 24 | } 25 | 26 | return item.buildRecord(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/ItemCountingItemProcessListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import java.util.Collection; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.batch.core.ItemProcessListener; 8 | 9 | @Slf4j 10 | @RequiredArgsConstructor 11 | public class ItemCountingItemProcessListener implements ItemProcessListener { 12 | 13 | private final AtomicLong itemsCounter; 14 | 15 | /* no op */ 16 | @Override 17 | public void beforeProcess(Object item) {} 18 | 19 | @Override 20 | public void afterProcess(Object item, Object result) { 21 | if (result == null) { 22 | return; 23 | } 24 | if (Collection.class.isAssignableFrom(result.getClass())) { 25 | itemsCounter.addAndGet(((Collection) result).size()); 26 | } else { 27 | itemsCounter.addAndGet(1L); 28 | } 29 | } 30 | 31 | @Override 32 | public void onProcessError(Object item, Exception e) { 33 | log.error(String.format("error while processing %s >> %s", item.toString(), e.getMessage())); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/master/MasterMainReleaseXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.master; 2 | 3 | import io.dsub.discogs.batch.domain.BaseXML; 4 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 5 | import java.time.Clock; 6 | import java.time.LocalDateTime; 7 | import javax.xml.bind.annotation.XmlAccessType; 8 | import javax.xml.bind.annotation.XmlAccessorType; 9 | import javax.xml.bind.annotation.XmlAttribute; 10 | import javax.xml.bind.annotation.XmlElement; 11 | import javax.xml.bind.annotation.XmlRootElement; 12 | import lombok.Data; 13 | import lombok.EqualsAndHashCode; 14 | 15 | @Data 16 | @XmlRootElement(name = "master") 17 | @XmlAccessorType(XmlAccessType.FIELD) 18 | @EqualsAndHashCode(callSuper = false) 19 | public class MasterMainReleaseXML implements BaseXML { 20 | 21 | @XmlAttribute(name = "id") 22 | private Integer id; 23 | 24 | @XmlElement(name = "main_release") 25 | private Integer mainReleaseId; 26 | 27 | @Override 28 | public MasterRecord buildRecord() { 29 | return new MasterRecord() 30 | .setId(id) 31 | .setMainReleaseId(mainReleaseId) 32 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/HashXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain; 2 | 3 | import org.jooq.UpdatableRecord; 4 | 5 | /** 6 | * A contract to enforce containing getHashValue() method on top of getRecord(int parentId). 7 | * 8 | * @param a subclass of {@link UpdatableRecord} that this class will produce as an instance. 9 | */ 10 | public interface HashXML> extends SubItemXML { 11 | 12 | int getHashValue(); 13 | 14 | /** 15 | * make hash values from given Strings. if values are null or empty, this will simply return the 16 | * hashcode from the instance. the same applies to the each value from the arguments. 17 | * 18 | * @param values to be hashed 19 | * @return object's hash if values are empty or null, else return hash from combined strings. 20 | */ 21 | default int makeHash(String[] values) { 22 | if (values == null || values.length == 0) { 23 | return hashCode(); 24 | } 25 | StringBuilder sb = new StringBuilder(); 26 | for (String v : values) { 27 | if (v != null && !v.isBlank()) { 28 | sb.append(v); 29 | } 30 | } 31 | return sb.isEmpty() ? this.hashCode() : sb.toString().hashCode(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/validator/KnownArgumentValidatorUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | 5 | import java.util.List; 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import org.springframework.boot.DefaultApplicationArguments; 9 | 10 | class KnownArgumentValidatorUnitTest { 11 | 12 | private KnownArgumentValidator validator; 13 | 14 | @BeforeEach 15 | void setUp() { 16 | validator = new KnownArgumentValidator(); 17 | } 18 | 19 | @Test 20 | void validate() { 21 | ValidationResult result = 22 | validator.validate( 23 | new DefaultApplicationArguments("--hello", "--world", "--string", "--chunk", "--t")); 24 | 25 | List issues = result.getIssues(); 26 | assertThat(issues.size()).isEqualTo(3); 27 | 28 | assertThat("unknown argument: string").isIn(issues); 29 | assertThat("unknown argument: hello").isIn(issues); 30 | assertThat("unknown argument: world").isIn(issues); 31 | 32 | assertThat("unknown argument: t").isNotIn(issues); 33 | assertThat("unknown argument: chunk").isNotIn(issues); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/ToggleProgressBarConsumer.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import java.io.PrintStream; 4 | import me.tongfei.progressbar.ConsoleProgressBarConsumer; 5 | 6 | /** 7 | * A console progress bar consumer that can be turned on or off. 8 | */ 9 | public class ToggleProgressBarConsumer extends ConsoleProgressBarConsumer { 10 | 11 | private boolean print = false; 12 | 13 | /** 14 | * Constructor to be used with designated {@link PrintStream}. 15 | * 16 | * @param out {@link PrintStream} to print progress bar. 17 | */ 18 | public ToggleProgressBarConsumer(PrintStream out) { 19 | super(out, 150); 20 | } 21 | 22 | /** 23 | * Regardless of acceptance, the act of print will be judged by either {@link 24 | * ToggleProgressBarConsumer#print} is on or off. 25 | */ 26 | @Override 27 | public void accept(String str) { 28 | if (this.print) { 29 | super.accept(str); 30 | } 31 | } 32 | 33 | /** 34 | * Simple on method to activate print. 35 | */ 36 | public void on() { 37 | this.print = true; 38 | } 39 | 40 | /** 41 | * Simple off method to deactivate print. 42 | */ 43 | public void off() { 44 | this.print = false; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/writer/DefaultJooqMasterMainReleaseItemWriter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.writer; 2 | 3 | import io.dsub.discogs.jooq.tables.Master; 4 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | import lombok.RequiredArgsConstructor; 8 | import org.jooq.DSLContext; 9 | import org.jooq.Query; 10 | 11 | @RequiredArgsConstructor 12 | public class DefaultJooqMasterMainReleaseItemWriter implements JooqItemWriter { 13 | 14 | private final DSLContext context; 15 | 16 | @Override 17 | public void write(List items) { 18 | 19 | if (items.isEmpty()) { 20 | return; 21 | } 22 | 23 | List updates = new LinkedList<>(); 24 | items.forEach(record -> updates.add(getQuery(record))); 25 | context.batch(updates).execute(); 26 | } 27 | 28 | @Override 29 | public Query getQuery(MasterRecord record) { 30 | return context 31 | .update(Master.MASTER) 32 | .set(Master.MASTER.LAST_MODIFIED_AT, record.getLastModifiedAt()) 33 | .set(Master.MASTER.MAIN_RELEASE_ID, record.getMainReleaseId()) 34 | .where(Master.MASTER.ID.eq(record.getId())); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/writer/CollectionItemWriter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.writer; 2 | 3 | import java.util.Collection; 4 | import java.util.HashMap; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | import java.util.Map; 8 | import lombok.RequiredArgsConstructor; 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.springframework.batch.item.ItemWriter; 11 | 12 | @Slf4j 13 | @RequiredArgsConstructor 14 | public class CollectionItemWriter implements ItemWriter> { 15 | 16 | private final ItemWriter delegate; 17 | 18 | @Override 19 | public void write(List> items) throws Exception { 20 | Map, List> consolidatedMap = new HashMap<>(); 21 | 22 | for (Collection subItems : items) { 23 | for (T subItem : subItems) { 24 | Class key = subItem.getClass(); 25 | if (!consolidatedMap.containsKey(subItem.getClass())) { 26 | consolidatedMap.put(key, new LinkedList<>()); 27 | } 28 | consolidatedMap.get(key).add(subItem); 29 | } 30 | } 31 | 32 | for (List subItems : consolidatedMap.values()) { 33 | delegate.write(subItems); 34 | subItems.clear(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/BatchService.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import io.dsub.discogs.batch.argument.handler.ArgumentHandler; 4 | import io.dsub.discogs.batch.argument.handler.DefaultArgumentHandler; 5 | import java.util.Arrays; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.context.ConfigurableApplicationContext; 9 | 10 | @Slf4j 11 | public class BatchService { 12 | 13 | protected ConfigurableApplicationContext run(String[] args) throws Exception { 14 | String[] resolved = resolveArguments(args); 15 | if (resolved == null) { 16 | log.info("Exiting..."); 17 | System.exit(1); 18 | } 19 | return runSpringApplication(resolveArguments(args)); 20 | } 21 | 22 | protected ConfigurableApplicationContext runSpringApplication(String[] args) { 23 | return SpringApplication.run(BatchApplication.class, args); 24 | } 25 | 26 | protected String[] resolveArguments(String[] args) { 27 | try { 28 | return getArgumentHandler().resolve(args); 29 | } catch (Exception e) { 30 | return null; 31 | } 32 | } 33 | 34 | protected ArgumentHandler getArgumentHandler() { 35 | return new DefaultArgumentHandler(); 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/KnownArgumentValidator.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import io.dsub.discogs.batch.argument.ArgType; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | import lombok.NoArgsConstructor; 7 | import org.springframework.boot.ApplicationArguments; 8 | 9 | /** 10 | * An argument validator implementation to check if we got unknown argument. 11 | */ 12 | @NoArgsConstructor 13 | public class KnownArgumentValidator implements ArgumentValidator { 14 | 15 | /** 16 | * Validation to check if all arguments are something we already know of. 17 | * 18 | * @param applicationArguments to be validated. 19 | * @return empty validation result if all arguments are known. Otherwise, returns a validation 20 | * result with reports of unknown arguments. 21 | */ 22 | @Override 23 | public ValidationResult validate(ApplicationArguments applicationArguments) { 24 | List issueList = applicationArguments.getOptionNames().stream() 25 | .filter(name -> !ArgType.contains(name.toLowerCase())) 26 | .map(name -> "unknown argument: " + name) 27 | .collect(Collectors.toList()); 28 | 29 | return new DefaultValidationResult(issueList); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/repository/DiscogsDumpRepository.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump.repository; 2 | 3 | import io.dsub.discogs.batch.dump.DiscogsDump; 4 | import io.dsub.discogs.batch.dump.EntityType; 5 | import java.time.LocalDate; 6 | import java.util.Collection; 7 | import java.util.List; 8 | import org.springframework.beans.factory.InitializingBean; 9 | 10 | public interface DiscogsDumpRepository extends InitializingBean { 11 | 12 | List findAllByLastModifiedAtIsBetween(LocalDate start, LocalDate end); 13 | 14 | List findAll(); 15 | 16 | int countItemsAfter(LocalDate start); 17 | 18 | int countItemsBefore(LocalDate end); 19 | 20 | int countItemsBetween(LocalDate start, LocalDate end); 21 | 22 | List findByTypeAndLastModifiedAtBetween( 23 | EntityType type, LocalDate start, LocalDate end); 24 | 25 | DiscogsDump findTopByTypeAndLastModifiedAtBetween( 26 | EntityType type, LocalDate start, LocalDate end); 27 | 28 | DiscogsDump findTopByType(EntityType type); 29 | 30 | DiscogsDump findByETag(String ETag); 31 | 32 | boolean existsByETag(String ETag); 33 | 34 | int count(); 35 | 36 | void saveAll(Collection discogsDumps); 37 | 38 | void deleteAll(); 39 | 40 | void save(DiscogsDump dump); 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/ProgressBarUtil.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import java.time.temporal.ChronoUnit; 4 | import me.tongfei.progressbar.ConsoleProgressBarConsumer; 5 | import me.tongfei.progressbar.ProgressBar; 6 | import me.tongfei.progressbar.ProgressBarBuilder; 7 | import me.tongfei.progressbar.ProgressBarConsumer; 8 | import me.tongfei.progressbar.ProgressBarStyle; 9 | 10 | /** 11 | * A convenient class to create {@link ProgressBar}. In needs of further customization, use {@link 12 | * ProgressBarBuilder}. 13 | */ 14 | public class ProgressBarUtil { 15 | 16 | // prevent initialize 17 | private ProgressBarUtil() { 18 | } 19 | 20 | public static ProgressBar get(String taskName, long initialMax) { 21 | return get(taskName, initialMax, new ConsoleProgressBarConsumer(System.err, 150)); 22 | } 23 | 24 | public static ProgressBar get(String taskName, long initialMax, ProgressBarConsumer consumer) { 25 | return new ProgressBarBuilder() 26 | .setStyle(ProgressBarStyle.COLORFUL_UNICODE_BLOCK) 27 | .setUnit("MB", 1048576) 28 | .setSpeedUnit(ChronoUnit.SECONDS) 29 | .setUpdateIntervalMillis(100) 30 | .setTaskName(taskName) 31 | .setInitialMax(initialMax) 32 | .setConsumer(consumer) 33 | .showSpeed() 34 | .build(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/ClearanceJobExecutionListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import io.dsub.discogs.batch.exception.FileException; 4 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 5 | import io.dsub.discogs.batch.util.FileUtil; 6 | import lombok.RequiredArgsConstructor; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.springframework.batch.core.JobExecution; 9 | import org.springframework.batch.core.JobExecutionListener; 10 | 11 | @Slf4j 12 | @RequiredArgsConstructor 13 | public class ClearanceJobExecutionListener implements JobExecutionListener { 14 | 15 | private final EntityIdRegistry registry; 16 | private final FileUtil fileUtil; 17 | 18 | @Override 19 | public void beforeJob(JobExecution jobExecution) { 20 | } 21 | 22 | @Override 23 | public void afterJob(JobExecution jobExecution) { 24 | clearCache(); 25 | clearFiles(); 26 | } 27 | 28 | private void clearFiles() { 29 | if (fileUtil.isTemporary()) { 30 | try { 31 | fileUtil.clearAll(); 32 | } catch (FileException e) { 33 | log.error("failed to clear application directory", e); 34 | } 35 | } else { 36 | log.info("mount option applied. skipping file deletion"); 37 | } 38 | } 39 | 40 | private void clearCache() { 41 | registry.clearAll(); 42 | log.info("cache cleared"); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/service/DefaultDatabaseConnectionValidatorIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.service; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import io.dsub.discogs.batch.argument.validator.DatabaseConnectionValidator; 6 | import io.dsub.discogs.batch.argument.validator.DefaultDatabaseConnectionValidator; 7 | import io.dsub.discogs.batch.argument.validator.ValidationResult; 8 | import io.dsub.discogs.batch.container.PostgreSQLContainerBaseTest; 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.junit.jupiter.api.Nested; 11 | import org.junit.jupiter.api.Test; 12 | 13 | @Slf4j 14 | public class DefaultDatabaseConnectionValidatorIntegrationTest { 15 | 16 | static final DatabaseConnectionValidator service = new DefaultDatabaseConnectionValidator(); 17 | 18 | @Nested 19 | class PostgreSQLIntegrationTest extends PostgreSQLContainerBaseTest { 20 | 21 | @Test 22 | void shouldPassIfCredentialsMatch() { 23 | // when 24 | ValidationResult result = service.validate(jdbcUrl, username, password); 25 | 26 | // then 27 | assertThat(result.isValid()).isTrue(); 28 | } 29 | 30 | @Test 31 | void shouldNotPassIfCredentialsNotMatch() { 32 | // when 33 | ValidationResult result = service.validate(jdbcUrl, "hello", password); 34 | 35 | // then 36 | assertThat(result.isValid()).isFalse(); 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/artist/ArtistXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.artist; 2 | 3 | import io.dsub.discogs.batch.domain.BaseXML; 4 | import io.dsub.discogs.jooq.tables.records.ArtistRecord; 5 | import java.time.Clock; 6 | import java.time.LocalDateTime; 7 | import javax.xml.bind.annotation.XmlAccessType; 8 | import javax.xml.bind.annotation.XmlAccessorType; 9 | import javax.xml.bind.annotation.XmlElement; 10 | import javax.xml.bind.annotation.XmlRootElement; 11 | import lombok.Data; 12 | 13 | @Data 14 | @XmlRootElement(name = "artist") 15 | @XmlAccessorType(XmlAccessType.FIELD) 16 | public class ArtistXML implements BaseXML { 17 | 18 | @XmlElement(name = "id") 19 | private Integer id; 20 | 21 | @XmlElement(name = "name") 22 | private String name; 23 | 24 | @XmlElement(name = "realname") 25 | private String realName; 26 | 27 | @XmlElement(name = "profile") 28 | private String profile; 29 | 30 | @XmlElement(name = "data_quality") 31 | private String dataQuality; 32 | 33 | @Override 34 | public ArtistRecord buildRecord() { 35 | return new ArtistRecord() 36 | .setId(id) 37 | .setName(name) 38 | .setRealName(realName) 39 | .setProfile(profile) 40 | .setDataQuality(dataQuality) 41 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())) 42 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/label/LabelXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.label; 2 | 3 | import io.dsub.discogs.batch.domain.BaseXML; 4 | import io.dsub.discogs.jooq.tables.records.LabelRecord; 5 | import java.time.Clock; 6 | import java.time.LocalDateTime; 7 | import javax.xml.bind.annotation.XmlAccessType; 8 | import javax.xml.bind.annotation.XmlAccessorType; 9 | import javax.xml.bind.annotation.XmlElement; 10 | import javax.xml.bind.annotation.XmlRootElement; 11 | import lombok.Data; 12 | 13 | @Data 14 | @XmlRootElement(name = "label") 15 | @XmlAccessorType(XmlAccessType.FIELD) 16 | public class LabelXML implements BaseXML { 17 | 18 | @XmlElement(name = "id") 19 | private Integer id; 20 | 21 | @XmlElement(name = "name") 22 | private String name; 23 | 24 | @XmlElement(name = "contactinfo") 25 | private String contactInfo; 26 | 27 | @XmlElement(name = "profile") 28 | private String profile; 29 | 30 | @XmlElement(name = "data_quality") 31 | private String dataQuality; 32 | 33 | @Override 34 | public LabelRecord buildRecord() { 35 | return new LabelRecord() 36 | .setId(id) 37 | .setName(name) 38 | .setContactInfo(contactInfo) 39 | .setProfile(profile) 40 | .setDataQuality(dataQuality) 41 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 42 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/reader/DiscogsDumpItemReaderBuilder.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.reader; 2 | 3 | import io.dsub.discogs.batch.dump.DiscogsDump; 4 | import io.dsub.discogs.batch.util.FileUtil; 5 | import java.nio.file.Path; 6 | import lombok.RequiredArgsConstructor; 7 | import org.springframework.batch.item.support.SynchronizedItemStreamReader; 8 | import org.springframework.util.Assert; 9 | 10 | /** 11 | * Utility class that provides single static method {@link #build(Class, DiscogsDump)}. 12 | */ 13 | @RequiredArgsConstructor 14 | public class DiscogsDumpItemReaderBuilder { 15 | 16 | private final FileUtil fileUtil; 17 | 18 | public SynchronizedItemStreamReader build(Class mappedClass, DiscogsDump dump) 19 | throws Exception { 20 | Assert.notNull(dump.getFileName(), "fileName of DiscogsDump cannot be null"); 21 | Assert.notNull(dump.getType(), "type of DiscogsDump cannot be null"); 22 | 23 | Path filePath = fileUtil.getFilePath(dump.getFileName()); 24 | 25 | ProgressBarStaxEventItemReader delegate; 26 | delegate = 27 | new ProgressBarStaxEventItemReader<>(mappedClass, filePath, dump.getType().toString()); 28 | delegate.afterPropertiesSet(); 29 | 30 | SynchronizedItemStreamReader reader = new SynchronizedItemStreamReader<>(); 31 | reader.setDelegate(delegate); 32 | reader.afterPropertiesSet(); // this won't trigger that of delegate's. 33 | return reader; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/writer/ItemWriterConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.writer; 2 | 3 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 4 | import java.util.Collection; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.jooq.DSLContext; 8 | import org.jooq.UpdatableRecord; 9 | import org.springframework.batch.core.configuration.annotation.StepScope; 10 | import org.springframework.batch.item.ItemWriter; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | 14 | @Slf4j 15 | @Configuration 16 | @RequiredArgsConstructor 17 | public class ItemWriterConfig { 18 | 19 | private final DSLContext context; 20 | 21 | @Bean 22 | public ItemWriter> jooqItemWriter() { 23 | return new DefaultLJooqItemWriter<>(context); 24 | } 25 | 26 | @Bean 27 | public ItemWriter>> baseEntityCollectionItemWriter() { 28 | return getBaseEntityCollectionItemWriter(jooqItemWriter()); 29 | } 30 | 31 | @Bean 32 | @StepScope 33 | public ItemWriter postgresJooqMasterMainReleaseItemWriter() { 34 | return new DefaultJooqMasterMainReleaseItemWriter(context); 35 | } 36 | 37 | private CollectionItemWriter> getBaseEntityCollectionItemWriter( 38 | ItemWriter> delegate) { 39 | return new CollectionItemWriter<>(delegate); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/step/AbstractStepConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.step; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.batch.core.job.flow.FlowExecutionStatus; 5 | import org.springframework.batch.core.job.flow.JobExecutionDecider; 6 | 7 | @Slf4j 8 | public abstract class AbstractStepConfig { 9 | 10 | protected static final String CHUNK = "#{jobParameters['chunkSize']}"; 11 | protected static final String ANY = "*"; 12 | protected static final String FAILED = "FAILED"; 13 | protected static final String SKIPPED = "SKIPPED"; 14 | protected static final String ARTIST = "artist"; 15 | protected static final String LABEL = "label"; 16 | protected static final String MASTER = "master"; 17 | protected static final String RELEASE = "release"; 18 | 19 | protected JobExecutionDecider executionDecider(String etagKey) { 20 | return (jobExecution, stepExecution) -> { 21 | if (jobExecution.getExitStatus().getExitCode().equals("FAILED")) { 22 | log.info("job execution marked as failed. skipping {} step", etagKey); 23 | return new FlowExecutionStatus(SKIPPED); 24 | } 25 | if (jobExecution.getJobParameters().getParameters().containsKey(etagKey)) { 26 | log.info("{} eTag found. executing {} step.", etagKey, etagKey); 27 | return FlowExecutionStatus.COMPLETED; 28 | } 29 | log.info("{} eTag not found. skipping {} step.", etagKey, etagKey); 30 | return new FlowExecutionStatus(SKIPPED); 31 | }; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/condition/RequiresDiscogsDataConnection.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.condition; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.net.MalformedURLException; 6 | import java.net.URL; 7 | import java.net.URLConnection; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.junit.jupiter.api.extension.ConditionEvaluationResult; 10 | import org.junit.jupiter.api.extension.ExecutionCondition; 11 | import org.junit.jupiter.api.extension.ExtensionContext; 12 | 13 | @Slf4j 14 | public class RequiresDiscogsDataConnection implements ExecutionCondition { 15 | 16 | private static final String DISCOGS_DATA_URL = "https://data.discogs.com"; 17 | 18 | @Override 19 | public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext ctx) { 20 | InputStream in = null; 21 | try { 22 | final URL url = new URL(DISCOGS_DATA_URL); 23 | final URLConnection conn = url.openConnection(); 24 | conn.connect(); 25 | in = conn.getInputStream(); 26 | } catch (MalformedURLException e) { 27 | // throw runtime exception as malformed url is an unacceptable fault. 28 | throw new RuntimeException(e); 29 | } catch (IOException e) { 30 | return ConditionEvaluationResult.disabled("failed to connect to URL: " + DISCOGS_DATA_URL); 31 | } finally { 32 | if (in != null) { 33 | try { 34 | in.close(); 35 | in = null; 36 | } catch (IOException ignored) { 37 | // ignored 38 | } 39 | } 40 | } 41 | return ConditionEvaluationResult.enabled("successfully connected to URL:" + DISCOGS_DATA_URL); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/ValidationResult.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import java.util.Collection; 4 | import java.util.List; 5 | 6 | /** 7 | * Represented result of a validation (mainly by {@link ArgumentValidator}). actual behavior depends 8 | * on its implementations. 9 | */ 10 | public interface ValidationResult { 11 | 12 | /** 13 | * Wither pattern to support adding a single issue. 14 | * 15 | * @param issue to be added. 16 | * @return accumulated, new instance. 17 | */ 18 | ValidationResult withIssue(String issue); 19 | 20 | /** 21 | * Wither pattern to support adding multiple issues without bound. 22 | * 23 | * @param issues to be added. 24 | * @return accumulated, new instance. 25 | */ 26 | ValidationResult withIssues(String... issues); 27 | 28 | /** 29 | * Wither pattern to support adding a collection of issues. 30 | * 31 | * @param issues to be added. 32 | * @return accumulated, new instance. 33 | */ 34 | ValidationResult withIssues(Collection issues); 35 | 36 | /** 37 | * Wither pattern to support adding another instance. 38 | * 39 | * @param other instance to be accumulated. 40 | * @return accumulated, new instance. 41 | */ 42 | ValidationResult combine(ValidationResult other); 43 | 44 | /** 45 | * Validation check to be added. The actual behavior will be depends on the implementation. 46 | * 47 | * @return evaluation of current accumulated result. 48 | */ 49 | boolean isValid(); 50 | 51 | /** 52 | * A support method to fetch accumulated issues. 53 | * 54 | * @return list of issues. 55 | */ 56 | List getIssues(); 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/tasklet/FileClearTasklet.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.tasklet; 2 | 3 | import io.dsub.discogs.batch.exception.FileException; 4 | import io.dsub.discogs.batch.util.FileUtil; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.batch.core.ExitStatus; 8 | import org.springframework.batch.core.StepContribution; 9 | import org.springframework.batch.core.scope.context.ChunkContext; 10 | import org.springframework.batch.core.step.tasklet.Tasklet; 11 | import org.springframework.batch.repeat.RepeatStatus; 12 | 13 | /** 14 | * A basic implementation of {@link Tasklet} to perform file clear. If file exists, this tasklet 15 | * will try to delete file from path offered by DiscogsDump. 16 | */ 17 | @Slf4j 18 | @RequiredArgsConstructor 19 | public class FileClearTasklet implements Tasklet { 20 | 21 | private final FileUtil fileUtil; 22 | 23 | /** 24 | * Deletes given file from targetDump. 25 | * 26 | * @param contribution will report {@link ExitStatus#FAILED} if failed to delete the file. 27 | * @param chunkContext required for implementation but will not interact. 28 | * @return {@link RepeatStatus#FINISHED} even if failed to delete the given file. 29 | */ 30 | @Override 31 | public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { 32 | if (fileUtil.isTemporary()) { 33 | try { 34 | fileUtil.clearAll(); 35 | } catch (FileException e) { 36 | log.error("failed to clear application directory.", e); 37 | } 38 | } 39 | contribution.setExitStatus(ExitStatus.COMPLETED); 40 | chunkContext.setComplete(); 41 | return RepeatStatus.FINISHED; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/release/ReleaseItemXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.release; 2 | 3 | import java.util.List; 4 | import javax.xml.bind.annotation.XmlAccessType; 5 | import javax.xml.bind.annotation.XmlAccessorType; 6 | import javax.xml.bind.annotation.XmlAttribute; 7 | import javax.xml.bind.annotation.XmlElement; 8 | import javax.xml.bind.annotation.XmlElementWrapper; 9 | import javax.xml.bind.annotation.XmlRootElement; 10 | import javax.xml.bind.annotation.XmlValue; 11 | import lombok.Data; 12 | import lombok.EqualsAndHashCode; 13 | 14 | @Data 15 | @XmlRootElement(name = "release") 16 | @XmlAccessorType(XmlAccessType.FIELD) 17 | @EqualsAndHashCode(callSuper = false) 18 | public class ReleaseItemXML { 19 | 20 | @XmlAttribute(name = "id") 21 | private Integer id; 22 | 23 | @XmlAttribute(name = "status") 24 | private String status; 25 | 26 | @XmlElement(name = "title") 27 | private String title; 28 | 29 | @XmlElement(name = "country") 30 | private String country; 31 | 32 | @XmlElement(name = "notes") 33 | private String notes; 34 | 35 | @XmlElement(name = "data_quality") 36 | private String dataQuality; 37 | 38 | @XmlElement(name = "released") 39 | private String releaseDate; 40 | 41 | @XmlElement(name = "master_id") 42 | private Master master; 43 | 44 | @XmlElementWrapper(name = "genres") 45 | @XmlElement(name = "genre") 46 | private List genres; 47 | 48 | @XmlElementWrapper(name = "styles") 49 | @XmlElement(name = "style") 50 | private List styles; 51 | 52 | @Data 53 | @XmlAccessorType(XmlAccessType.FIELD) 54 | public static class Master { 55 | 56 | @XmlValue 57 | Integer masterId; 58 | 59 | @XmlAttribute(name = "is_main_release") 60 | boolean isMaster; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/FileUtil.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import io.dsub.discogs.batch.exception.FileDeleteException; 4 | import io.dsub.discogs.batch.exception.FileException; 5 | import java.io.InputStream; 6 | import java.nio.file.Path; 7 | import org.apache.commons.lang3.SystemUtils; 8 | 9 | /** 10 | * Interface to support file creation, deletion operations only within application directory. 11 | * Currently, there is no need to utilize second level, hence flattening the method params by 12 | * String. There is NO GUARANTEE that this interface will not grow in the future. 13 | * 14 | *

Methods and their behaviors will depends on its implementations. 15 | */ 16 | public interface FileUtil { 17 | 18 | String DEFAULT_APP_DIR = "discogs-data-batch"; 19 | 20 | void clearAll() throws FileException; 21 | 22 | Path getFilePath(String filename, boolean generate) throws FileException; 23 | 24 | Path getFilePath(String filename) throws FileException; 25 | 26 | Path getAppDirectory(boolean generate) throws FileException; 27 | 28 | void deleteFile(String filename) throws FileDeleteException; 29 | 30 | boolean isExisting(String filename); 31 | 32 | long getSize(String filename) throws FileException; 33 | 34 | void copy(InputStream inputStream, String filename) throws FileException; 35 | 36 | /** 37 | * A wrapper method to get home directory from the parent OS. The implementation of the method may 38 | * change over time, in case of bugs or requirements of additional logics. 39 | * 40 | * @return directory promised to be home directory (i.e. `~`.) 41 | */ 42 | default Path getHomeDirectory() { 43 | return SystemUtils.getUserHome().toPath(); 44 | } 45 | 46 | String getAppDirectory(); 47 | 48 | boolean isTemporary(); 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/master/MasterXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.master; 2 | 3 | import io.dsub.discogs.batch.domain.BaseXML; 4 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 5 | import java.time.Clock; 6 | import java.time.LocalDateTime; 7 | import java.util.List; 8 | import javax.xml.bind.annotation.XmlAccessType; 9 | import javax.xml.bind.annotation.XmlAccessorType; 10 | import javax.xml.bind.annotation.XmlAttribute; 11 | import javax.xml.bind.annotation.XmlElement; 12 | import javax.xml.bind.annotation.XmlElementWrapper; 13 | import javax.xml.bind.annotation.XmlRootElement; 14 | import lombok.Data; 15 | import lombok.EqualsAndHashCode; 16 | 17 | @Data 18 | @XmlRootElement(name = "master") 19 | @XmlAccessorType(XmlAccessType.FIELD) 20 | @EqualsAndHashCode(callSuper = false) 21 | public class MasterXML implements BaseXML { 22 | 23 | @XmlAttribute(name = "id") 24 | private Integer id; 25 | 26 | @XmlElement(name = "year") 27 | private Short year; 28 | 29 | @XmlElement(name = "title") 30 | private String title; 31 | 32 | @XmlElement(name = "main_release") 33 | private Integer mainReleaseId; 34 | 35 | @XmlElement(name = "data_quality") 36 | private String dataQuality; 37 | 38 | @XmlElementWrapper(name = "genres") 39 | @XmlElement(name = "genre") 40 | private List genres; 41 | 42 | @XmlElementWrapper(name = "styles") 43 | @XmlElement(name = "style") 44 | private List styles; 45 | 46 | @Override 47 | public MasterRecord buildRecord() { 48 | return new MasterRecord() 49 | .setId(id) 50 | .setTitle(title) 51 | .setYear(year) 52 | .setDataQuality(dataQuality) 53 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 54 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/JobLaunchingRunner.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import java.util.concurrent.CountDownLatch; 4 | import lombok.RequiredArgsConstructor; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.batch.core.Job; 7 | import org.springframework.batch.core.JobExecution; 8 | import org.springframework.batch.core.JobParameters; 9 | import org.springframework.batch.core.launch.JobLauncher; 10 | import org.springframework.boot.ApplicationArguments; 11 | import org.springframework.boot.ApplicationRunner; 12 | import org.springframework.boot.ExitCodeGenerator; 13 | import org.springframework.boot.SpringApplication; 14 | import org.springframework.context.ConfigurableApplicationContext; 15 | import org.springframework.context.annotation.Profile; 16 | import org.springframework.core.annotation.Order; 17 | import org.springframework.stereotype.Component; 18 | 19 | @Slf4j 20 | @Order(10) 21 | @Profile("!test") 22 | @Component 23 | @RequiredArgsConstructor 24 | public class JobLaunchingRunner implements ApplicationRunner { 25 | 26 | private final Job job; 27 | private final JobParameters discogsJobParameters; 28 | private final JobLauncher jobLauncher; 29 | private final ConfigurableApplicationContext ctx; 30 | private final CountDownLatch countDownLatch; 31 | 32 | @Override 33 | public void run(ApplicationArguments args) throws Exception { 34 | JobExecution jobExecution = jobLauncher.run(job, discogsJobParameters); 35 | log.info("main thread started job execution. awaiting for completion..."); 36 | countDownLatch.await(); 37 | log.info("job execution completed. exiting..."); 38 | SpringApplication.exit(ctx, getExitCodeGenerator(jobExecution)); 39 | } 40 | 41 | public ExitCodeGenerator getExitCodeGenerator(JobExecution jobExecution) { 42 | return () -> jobExecution.getFailureExceptions().size() > 0 ? 1 : 0; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/formatter/ArgumentNameFormatter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | import io.dsub.discogs.batch.argument.ArgType; 4 | import java.util.Arrays; 5 | 6 | /** 7 | * ArgumentFormatter that formats argument's name 8 | */ 9 | public class ArgumentNameFormatter implements ArgumentFormatter { 10 | 11 | /** 12 | * Formats argument name so that it does not contains any plurals. Also, it will check the 13 | * argument name and assign proper name from {@link ArgType}. 14 | * 15 | * @param args argument to be evaluated. 16 | * @return argument with formatted name, or as-is if name is unrecognizable. 17 | */ 18 | @Override 19 | public String[] format(String[] args) { 20 | return Arrays.stream(args) 21 | .map(this::doFormat) 22 | .toArray(String[]::new); 23 | } 24 | 25 | private String doFormat(String arg) { 26 | // make lower case 27 | String head = arg.split("=")[0].toLowerCase(); 28 | // remove all plurals('[sS]$') 29 | if (head.matches(".*[sS]$") && !ArgType.contains(head)) { 30 | head = head.substring(0, head.length() - 1); 31 | } 32 | // fetch type of the argument name. 33 | ArgType type = ArgType.getTypeOf(head); 34 | 35 | String value; 36 | 37 | // meaning the last character is equals sign 38 | if (arg.indexOf('=') == arg.length() - 1 || arg.indexOf('=') == -1) { 39 | value = ""; 40 | } else { 41 | // parse the value string from given argument. 42 | value = arg.substring(arg.indexOf("=") + 1); 43 | } 44 | 45 | // unknown type, hence return argument as-is. 46 | if (type == null) { 47 | return arg; 48 | } 49 | 50 | head = type.getGlobalName(); 51 | 52 | // handle no-arg option 53 | if (!type.isValueRequired()) { 54 | return head; 55 | } 56 | 57 | return String.join("=", head, value); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/tasklet/GenreStyleInsertionTasklet.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.tasklet; 2 | 3 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 4 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 5 | import io.dsub.discogs.jooq.tables.records.GenreRecord; 6 | import io.dsub.discogs.jooq.tables.records.StyleRecord; 7 | import java.util.stream.Collectors; 8 | import lombok.RequiredArgsConstructor; 9 | import org.jooq.UpdatableRecord; 10 | import org.springframework.batch.core.ExitStatus; 11 | import org.springframework.batch.core.StepContribution; 12 | import org.springframework.batch.core.scope.context.ChunkContext; 13 | import org.springframework.batch.core.step.tasklet.Tasklet; 14 | import org.springframework.batch.item.ItemWriter; 15 | import org.springframework.batch.repeat.RepeatStatus; 16 | import org.springframework.stereotype.Component; 17 | 18 | @Component 19 | @RequiredArgsConstructor 20 | public class GenreStyleInsertionTasklet implements Tasklet { 21 | 22 | private final EntityIdRegistry registry; 23 | private final ItemWriter> jooqItemWriter; 24 | 25 | @Override 26 | public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) 27 | throws Exception { 28 | contribution.setExitStatus(ExitStatus.EXECUTING); 29 | jooqItemWriter.write( 30 | registry.getStringIdSetByType(DefaultEntityIdRegistry.Type.GENRE).stream() 31 | .map(genre -> new GenreRecord().setName(genre)) 32 | .collect(Collectors.toList())); 33 | jooqItemWriter.write( 34 | registry.getStringIdSetByType(DefaultEntityIdRegistry.Type.STYLE).stream() 35 | .map(style -> new StyleRecord().setName(style)) 36 | .collect(Collectors.toList())); 37 | contribution.setExitStatus(ExitStatus.COMPLETED); 38 | chunkContext.setComplete(); 39 | return RepeatStatus.FINISHED; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/label/LabelSubItemsXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.label; 2 | 3 | import io.dsub.discogs.batch.domain.SubItemXML; 4 | import io.dsub.discogs.jooq.tables.records.LabelSubLabelRecord; 5 | import java.time.Clock; 6 | import java.time.LocalDateTime; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import javax.xml.bind.annotation.XmlAccessType; 10 | import javax.xml.bind.annotation.XmlAccessorType; 11 | import javax.xml.bind.annotation.XmlAttribute; 12 | import javax.xml.bind.annotation.XmlElement; 13 | import javax.xml.bind.annotation.XmlElementWrapper; 14 | import javax.xml.bind.annotation.XmlRootElement; 15 | import javax.xml.bind.annotation.XmlValue; 16 | import lombok.Data; 17 | import lombok.EqualsAndHashCode; 18 | 19 | @Data 20 | @XmlRootElement(name = "label") 21 | @XmlAccessorType(XmlAccessType.FIELD) 22 | @EqualsAndHashCode(callSuper = false) 23 | public class LabelSubItemsXML { 24 | 25 | @XmlElement(name = "id") 26 | private Integer id; 27 | 28 | @XmlElementWrapper(name = "sublabels") 29 | @XmlElement(name = "label") 30 | private List labelSubLabels = new ArrayList<>(); 31 | 32 | @XmlElementWrapper(name = "urls") 33 | @XmlElement(name = "url") 34 | private List urls = new ArrayList<>(); 35 | 36 | @Data 37 | @XmlAccessorType(XmlAccessType.FIELD) 38 | public static class LabelSubLabelXML implements SubItemXML { 39 | 40 | @XmlValue 41 | private String name; 42 | 43 | @XmlAttribute(name = "id") 44 | private Integer subLabelId; 45 | 46 | @Override 47 | public LabelSubLabelRecord getRecord(int parentId) { 48 | return new LabelSubLabelRecord() 49 | .setParentLabelId(parentId) 50 | .setSubLabelId(subLabelId) 51 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 52 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/TypeArgumentValidator.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import java.util.List; 4 | import java.util.regex.Pattern; 5 | import java.util.stream.Collectors; 6 | import org.apache.logging.log4j.util.Strings; 7 | import org.springframework.boot.ApplicationArguments; 8 | 9 | public class TypeArgumentValidator implements ArgumentValidator { 10 | 11 | private static final Pattern TYPE_VALUE_PATTERN = 12 | Pattern.compile("^((ARTIST)|(RELEASE)|(MASTER)|(LABEL))$", Pattern.CASE_INSENSITIVE); 13 | 14 | private static final Pattern TYPE_PATTERN = 15 | Pattern.compile("^type[s]?$", Pattern.CASE_INSENSITIVE); 16 | 17 | @Override 18 | public ValidationResult validate(ApplicationArguments args) { 19 | // init 20 | ValidationResult result = new DefaultValidationResult(); 21 | 22 | // collect possible duplicate type argument. 23 | List typeArgNames = 24 | args.getOptionNames().stream() 25 | .filter(name -> TYPE_PATTERN.matcher(name).matches()) 26 | .collect(Collectors.toList()); 27 | 28 | // empty means we do not need to validate anything. 29 | if (typeArgNames.isEmpty()) { 30 | return result; 31 | } 32 | 33 | // duplicated entries to be picked up from here. 34 | if (typeArgNames.size() > 1) { 35 | String msg = "duplicated type argument exists: " + Strings.join(typeArgNames, ','); 36 | return result.withIssues(msg); 37 | } 38 | 39 | // collect issues if any of argument does not match to the criteria. 40 | List issues = 41 | args.getOptionValues(typeArgNames.get(0)).stream() 42 | .filter(val -> !TYPE_VALUE_PATTERN.matcher(val).matches()) 43 | .map(val -> "unknown type argument value: " + val) 44 | .collect(Collectors.toList()); 45 | 46 | // will be empty if there was no issue. 47 | return result.withIssues(issues); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/container/PostgreSQLContainerBaseTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.container; 2 | 3 | import javax.sql.DataSource; 4 | import org.springframework.boot.jdbc.DataSourceBuilder; 5 | import org.springframework.boot.test.util.TestPropertyValues; 6 | import org.springframework.context.ApplicationContextInitializer; 7 | import org.springframework.context.ConfigurableApplicationContext; 8 | import org.testcontainers.containers.PostgreSQLContainer; 9 | 10 | public abstract class PostgreSQLContainerBaseTest { 11 | 12 | protected static final PostgreSQLContainer CONTAINER; 13 | protected static final DataSource dataSource; 14 | 15 | static { 16 | CONTAINER = new PostgreSQLContainer("postgres:latest") 17 | .withDatabaseName("databaseName") 18 | .withPassword("password") 19 | .withUsername("username"); 20 | CONTAINER.start(); 21 | dataSource = DataSourceBuilder.create() 22 | .driverClassName(CONTAINER.getDriverClassName()) 23 | .url(CONTAINER.getJdbcUrl()) 24 | .username(CONTAINER.getUsername()) 25 | .password(CONTAINER.getPassword()) 26 | .build(); 27 | } 28 | 29 | protected final String jdbcUrl = CONTAINER.getJdbcUrl(); 30 | protected final String password = CONTAINER.getPassword(); 31 | protected final String username = CONTAINER.getUsername(); 32 | 33 | static class PostgreSQLPropertiesInitializer implements 34 | ApplicationContextInitializer { 35 | 36 | @Override 37 | public void initialize(ConfigurableApplicationContext applicationContext) { 38 | TestPropertyValues.of("spring.datasource.driver-class-name=" + CONTAINER.getDriverClassName(), 39 | "spring.datasource.username=" + CONTAINER.getUsername(), 40 | "spring.datasource.password=" + CONTAINER.getPassword(), 41 | "spring.datasource.url=" + CONTAINER.getJdbcUrl()) 42 | .applyTo(applicationContext.getEnvironment()); 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/handler/DefaultArgumentHandlerIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.handler; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | 5 | import io.dsub.discogs.batch.container.PostgreSQLContainerBaseTest; 6 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 7 | import org.junit.jupiter.api.Assertions; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.params.ParameterizedTest; 10 | import org.junit.jupiter.params.provider.ValueSource; 11 | 12 | class DefaultArgumentHandlerIntegrationTest extends PostgreSQLContainerBaseTest { 13 | 14 | private final ArgumentHandler handler = new DefaultArgumentHandler(); 15 | 16 | @Test 17 | void shouldHandleMalformedUrlArgumentFlag() throws InvalidArgumentException { 18 | String[] args = new String[]{"url=" + jdbcUrl, "user=" + username, "pass=" + password}; 19 | Assertions.assertDoesNotThrow(() -> handler.resolve(args)); 20 | String[] resolved = handler.resolve(args); 21 | for (String s : resolved) { 22 | assertThat(s.startsWith("--")).isTrue(); 23 | } 24 | } 25 | 26 | @ParameterizedTest 27 | @ValueSource(strings = {"--m", "m", "--mount", "mount"}) 28 | void whenOptionArgGiven__ShouldAddAsOption__RegardlessOfDashPresented(String arg) 29 | throws InvalidArgumentException { 30 | String[] args = {"url=" + jdbcUrl, "user=" + username, "pass=" + password, arg}; 31 | args = handler.resolve(args); 32 | assertThat(args).contains("--mount"); 33 | } 34 | 35 | @Test 36 | void shouldReplacePlurals() throws InvalidArgumentException { 37 | String[] args = {"urls=" + jdbcUrl, "user=" + username, "pass=" + password, "etags=hello"}; 38 | args = handler.resolve(args); 39 | for (String arg : args) { 40 | String head = arg.split("=")[0]; 41 | if (head.contains("pass")) { 42 | continue; 43 | } 44 | assertThat(arg.split("=")[0].matches(".*s$")).isFalse(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/formatter/JdbcUrlFormatterTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import ch.qos.logback.classic.Level; 6 | import io.dsub.discogs.batch.testutil.LogSpy; 7 | import java.util.Arrays; 8 | import java.util.Optional; 9 | import org.junit.jupiter.api.Test; 10 | import org.junit.jupiter.api.extension.RegisterExtension; 11 | 12 | public class JdbcUrlFormatterTest { 13 | 14 | JdbcUrlFormatter formatter = new JdbcUrlFormatter(); 15 | 16 | @RegisterExtension 17 | LogSpy logSpy = new LogSpy(); 18 | 19 | @Test 20 | void whenSchemaOrDatabaseMissing__ShouldAddDefault() { 21 | String[] args = new String[]{"url=jdbc:postgresql://localhost:3306?serverTimeZone=UTC", 22 | "user=root", "pass=gozldwmf77", "m"}; 23 | 24 | String[] formatted = formatter.format(args); 25 | 26 | Optional urlArg = Arrays.stream(formatted) 27 | .filter(arg -> arg.startsWith("url")) 28 | .map(arg -> arg.replace("url=", "")) 29 | .findFirst(); 30 | 31 | assertThat(urlArg).isPresent(); 32 | 33 | String arg = urlArg.get(); 34 | 35 | assertThat(arg).matches(".*discogs.*"); 36 | 37 | assertThat(logSpy.getLogsByExactLevelAsString(Level.INFO, true)) 38 | .hasSize(1) 39 | .anyMatch(s -> s.matches("^default database or schema missing.*")); 40 | } 41 | 42 | @Test 43 | void whenArrayIsNull__ShouldReturnNull() { 44 | 45 | // when 46 | String[] formatted = formatter.format(null); 47 | 48 | // then 49 | assertThat(formatted).isNull(); 50 | } 51 | 52 | @Test 53 | void whenArrayIsEmpty__ShouldReturnEmptyArray() { 54 | String[] args = new String[0]; 55 | 56 | // when 57 | String[] formatted = formatter.format(args); 58 | 59 | // then 60 | assertThat(formatted) 61 | .isNotNull() 62 | .isEmpty(); 63 | } 64 | 65 | @Test 66 | void whenUrlIsEmpty__ShouldReturnNull() { 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/service/DefaultDatabaseConnectionValidatorUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.service; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import io.dsub.discogs.batch.argument.validator.DefaultDatabaseConnectionValidator; 6 | import io.dsub.discogs.batch.argument.validator.ValidationResult; 7 | import io.dsub.discogs.batch.testutil.LogSpy; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.extension.RegisterExtension; 10 | import org.springframework.boot.ApplicationArguments; 11 | import org.springframework.boot.DefaultApplicationArguments; 12 | 13 | class DefaultDatabaseConnectionValidatorUnitTest { 14 | 15 | final DefaultDatabaseConnectionValidator validator = new DefaultDatabaseConnectionValidator(); 16 | 17 | @RegisterExtension 18 | LogSpy logSpy = new LogSpy(); 19 | 20 | @Test 21 | void shouldPassIfValidDataSourceValueIsPresent() { 22 | ApplicationArguments args = getArgs("--url=jdbc:mysql://", "--password", "--username=sa"); 23 | 24 | // when 25 | ValidationResult validationResult = validator.validate(args); 26 | 27 | // then 28 | assertThat(validationResult.isValid()).isFalse(); 29 | } 30 | 31 | @Test 32 | void shouldReportIfInvalidValueHasBeenPassed() { 33 | // when 34 | ValidationResult result = validator 35 | .validate(getArgs("--url=hello", "--username=un", "--password=pw")); 36 | 37 | // then 38 | assertThat(result.getIssues()).contains("failed to allocate driver for url: hello"); 39 | } 40 | 41 | @Test 42 | void whenArgumentExceptionIfUrlMissing__ShouldReturnValidationResult() { 43 | 44 | // when 45 | ValidationResult result = validator.validate(getArgs("--password=something", "--username=un")); 46 | 47 | // then 48 | assertThat(result.isValid()).isFalse(); 49 | assertThat(result.getIssues()).contains("url cannot be null or blank"); 50 | } 51 | 52 | private ApplicationArguments getArgs(String... args) { 53 | return new DefaultApplicationArguments(args); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/writer/DefaultLJooqItemWriter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.writer; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.jooq.BatchBindStep; 8 | import org.jooq.DSLContext; 9 | import org.jooq.Field; 10 | import org.jooq.Query; 11 | import org.jooq.UpdatableRecord; 12 | 13 | @Slf4j 14 | @RequiredArgsConstructor 15 | public class DefaultLJooqItemWriter> extends AbstractJooqItemWriter { 16 | 17 | private final DSLContext context; 18 | 19 | @Override 20 | public void write(List items) { 21 | if (items.isEmpty()) { 22 | return; 23 | } 24 | Query q = this.getQuery(items.get(0)); 25 | BatchBindStep batch = context.batch(q); 26 | 27 | items.forEach(record -> batch.bind(mapValues(record))); 28 | batch.execute(); 29 | } 30 | 31 | /** 32 | * map values from record into a full array 33 | * 34 | * @param record to be parsed into array 35 | * @return values 36 | */ 37 | private Object[] mapValues(T record) { 38 | List values = getInsertValues(record); 39 | getUpdateFields(record.getTable()).forEach(field -> values.add(field.getValue(record))); 40 | return values.toArray(); 41 | } 42 | 43 | @Override 44 | public Query getQuery(T record) { 45 | List> constraintFields = getConstraintFields(record.getTable()); 46 | List> fieldsToUpdate = getUpdateFields(record.getTable()); 47 | Map updateMap = getUpdateMap(record); 48 | 49 | if (fieldsToUpdate.isEmpty()) { 50 | return context 51 | .insertInto(record.getTable(), getInsertFields(record.getTable())) 52 | .values(getInsertValues(record)) 53 | .onConflict(constraintFields) 54 | .doNothing(); 55 | } 56 | 57 | return context 58 | .insertInto(record.getTable(), getInsertFields(record.getTable())) 59 | .values(getInsertValues(record)) 60 | .onConflict(constraintFields) 61 | .doUpdate() 62 | .set(updateMap); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/decider/MasterMainReleaseStepJobExecutionDecider.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.decider; 2 | 3 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 4 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.batch.core.ExitStatus; 8 | import org.springframework.batch.core.JobExecution; 9 | import org.springframework.batch.core.StepExecution; 10 | import org.springframework.batch.core.job.flow.FlowExecutionStatus; 11 | import org.springframework.batch.core.job.flow.JobExecutionDecider; 12 | 13 | @Slf4j 14 | @RequiredArgsConstructor 15 | // TODO: test! 16 | public class MasterMainReleaseStepJobExecutionDecider implements JobExecutionDecider { 17 | 18 | private static final String MASTER = "master"; 19 | private static final String RELEASE_ITEM = "release"; 20 | private static final String SKIP_MSG = "skipping master main release step."; 21 | private static final String SKIPPED = "SKIPPED"; 22 | 23 | private final EntityIdRegistry idRegistry; 24 | 25 | @Override 26 | public FlowExecutionStatus decide(JobExecution jobExecution, StepExecution stepExecution) { 27 | if (jobExecution.getExitStatus().equals(ExitStatus.FAILED)) { 28 | log.info("job execution marked as failed. " + SKIP_MSG); 29 | } else if (stepExecution.getExitStatus().equals(ExitStatus.FAILED)) { 30 | log.info("step execution marked as failed. " + SKIP_MSG); 31 | } else if (!jobExecution.getJobParameters().getParameters().containsKey(MASTER)) { 32 | log.info("master eTag missing. " + SKIP_MSG); 33 | } else if (!jobExecution.getJobParameters().getParameters().containsKey(RELEASE_ITEM)) { 34 | log.info("release item eTag missing. " + SKIP_MSG); 35 | } else if (idRegistry 36 | .getLongIdCache(DefaultEntityIdRegistry.Type.RELEASE) 37 | .getConcurrentSkipListSet() 38 | .isEmpty()) { 39 | log.info("release item identity cache is missing. " + SKIP_MSG); 40 | } else { 41 | return FlowExecutionStatus.COMPLETED; 42 | } 43 | return new FlowExecutionStatus(SKIPPED); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/DataSourceUtil.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import io.dsub.discogs.batch.datasource.DBType; 4 | import io.dsub.discogs.batch.datasource.DataSourceDetails; 5 | import java.sql.Connection; 6 | import java.sql.DatabaseMetaData; 7 | import java.sql.SQLException; 8 | import java.util.regex.Pattern; 9 | import javax.sql.DataSource; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.jooq.SQLDialect; 12 | import org.springframework.util.Assert; 13 | 14 | /** 15 | * A utility class to reduce data source related repeated methods. 16 | */ 17 | @Slf4j 18 | public final class DataSourceUtil { 19 | 20 | public static final Pattern JDBC_URL_PATTERN = Pattern 21 | .compile("jdbc:\\w+://[.\\w]+:[\\d]+/(\\w+).*"); 22 | 23 | /* prevent instantiation */ 24 | private DataSourceUtil() { 25 | } 26 | 27 | public static DataSourceDetails getDataSourceDetails(DataSource dataSource) { 28 | DBType type = getDBTypeFrom(dataSource); 29 | Assert.notNull(type, "DBType cannot be null"); 30 | SQLDialect dialect = getSQLDialect(type); 31 | return new DataSourceDetails(dataSource, dialect, type); 32 | } 33 | 34 | public static SQLDialect getSQLDialect(DBType dbType) { 35 | return SQLDialect.POSTGRES; 36 | } 37 | 38 | public static DBType getDBTypeFrom(DataSource dataSource) { 39 | try (Connection conn = dataSource.getConnection()) { 40 | return DBType.getTypeOf(conn.getMetaData().getDatabaseProductName()); 41 | } catch (SQLException e) { 42 | log.error("failed to establish connection.", e); 43 | return null; 44 | } 45 | } 46 | 47 | public static DatabaseMetaData getMetaData(DataSource dataSource) { 48 | try (Connection conn = dataSource.getConnection()) { 49 | return conn.getMetaData(); 50 | } catch (SQLException e) { 51 | log.error("failed to establish connection.", e); 52 | return null; 53 | } 54 | } 55 | 56 | public static String getCatalogName(DataSource dataSource) { 57 | try (Connection conn = dataSource.getConnection()) { 58 | return conn.getCatalog(); 59 | } catch (SQLException e) { 60 | log.error("failed to establish connection.", e); 61 | return null; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/dump/repository/DiscogsDiscogsDumpRepositoryIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump.repository; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.junit.jupiter.api.Assertions.assertAll; 5 | 6 | import io.dsub.discogs.batch.condition.RequiresDiscogsDataConnection; 7 | import io.dsub.discogs.batch.dump.DefaultDumpSupplier; 8 | import io.dsub.discogs.batch.dump.DiscogsDump; 9 | import io.dsub.discogs.batch.dump.DumpSupplier; 10 | import io.dsub.discogs.batch.dump.EntityType; 11 | import java.util.List; 12 | import org.junit.jupiter.api.BeforeAll; 13 | import org.junit.jupiter.api.Test; 14 | import org.junit.jupiter.api.extension.ExtendWith; 15 | import org.junit.jupiter.params.ParameterizedTest; 16 | import org.junit.jupiter.params.provider.EnumSource; 17 | 18 | @ExtendWith(RequiresDiscogsDataConnection.class) 19 | class DiscogsDiscogsDumpRepositoryIntegrationTest { 20 | 21 | static DumpSupplier dumpSupplier; 22 | static DiscogsDumpRepository repository; 23 | 24 | @BeforeAll 25 | static void beforeAll() throws Exception { 26 | dumpSupplier = new DefaultDumpSupplier(); 27 | MapDiscogsDumpRepository mapDiscogsDumpRepository = new MapDiscogsDumpRepository(dumpSupplier); 28 | mapDiscogsDumpRepository.afterPropertiesSet(); 29 | repository = mapDiscogsDumpRepository; 30 | } 31 | 32 | @Test 33 | void whenFindAll__ShouldNotReturnEmptyList() { 34 | // when 35 | List found = repository.findAll(); 36 | 37 | // then 38 | assertThat(found).isNotEmpty(); 39 | } 40 | 41 | @ParameterizedTest 42 | @EnumSource(EntityType.class) 43 | void whenFindTopByType__ShouldReturnDiscogsDumpWithValidValues(EntityType type) { 44 | // when 45 | DiscogsDump dump = repository.findTopByType(type); 46 | 47 | // then 48 | assertAll( 49 | () -> assertThat(dump.getLastModifiedAt()).isNotNull(), 50 | () -> assertThat(dump.getETag()).isNotNull(), 51 | () -> assertThat(dump.getSize()).isNotNull(), 52 | () -> assertThat(dump.getUriString()).isNotNull(), 53 | () -> assertThat(dump.getFileName()).isNotNull(), 54 | () -> assertThat(dump.getType()).isNotNull(), 55 | () -> assertThat(dump.getUrl()).isNotNull() 56 | ); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/dump/DiscogsDump.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.dump; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.net.URL; 6 | import java.time.LocalDate; 7 | import java.util.Objects; 8 | import lombok.Data; 9 | import lombok.RequiredArgsConstructor; 10 | 11 | @Data 12 | @RequiredArgsConstructor 13 | public class DiscogsDump implements Comparable { 14 | 15 | private final String eTag; 16 | private final EntityType type; 17 | private final String uriString; 18 | private final Long size; 19 | private final LocalDate lastModifiedAt; 20 | private final URL url; 21 | 22 | public InputStream getInputStream() throws IOException { 23 | if (this.url == null) { 24 | return InputStream.nullInputStream(); 25 | } 26 | return this.url.openStream(); 27 | } 28 | 29 | // parse file name from the uriString formatted as data/{year}/{file_name}; 30 | public String getFileName() { 31 | if (this.uriString == null || this.uriString.isBlank()) { 32 | return null; 33 | } 34 | return this.uriString.substring(this.uriString.lastIndexOf('/') + 1); 35 | } 36 | 37 | @Override 38 | public int compareTo(DiscogsDump that) { 39 | int res = this.lastModifiedAt.compareTo(that.lastModifiedAt); 40 | if (res != 0) { 41 | return res; 42 | } 43 | res = this.type.compareTo(that.getType()); 44 | if (res != 0) { 45 | return res; 46 | } 47 | res = this.eTag.compareTo(that.getETag()); 48 | if (res != 0) { 49 | return res; 50 | } 51 | return this.size.compareTo(that.getSize()); 52 | } 53 | 54 | /** 55 | * Compare only equals with ETag value as it is the single most definite identification of a 56 | * dump. 57 | * 58 | * @param o any object, or another instance of dump to be evaluated being equal. 59 | * @return the result of the equals method. 60 | */ 61 | @Override 62 | public boolean equals(Object o) { 63 | if (this == o) { 64 | return true; 65 | } 66 | if (o == null || getClass() != o.getClass()) { 67 | return false; 68 | } 69 | DiscogsDump that = (DiscogsDump) o; 70 | return eTag.equals(that.eTag); 71 | } 72 | 73 | @Override 74 | public int hashCode() { 75 | return Objects.hash(eTag); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/BatchListenerConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 4 | import io.dsub.discogs.batch.util.FileUtil; 5 | import java.util.concurrent.CountDownLatch; 6 | import java.util.concurrent.atomic.AtomicLong; 7 | import lombok.RequiredArgsConstructor; 8 | import org.jooq.DSLContext; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | @Configuration 13 | @RequiredArgsConstructor 14 | public class BatchListenerConfig { 15 | 16 | @Bean 17 | public DefaultEntityIdRegistry entityIdRegistry() { 18 | return new DefaultEntityIdRegistry(); 19 | } 20 | 21 | @Bean 22 | public AtomicLong itemsCounter() { 23 | return new AtomicLong(); 24 | } 25 | 26 | @Bean 27 | public IdCachingItemProcessListener idCachingItemProcessListener() { 28 | return new IdCachingItemProcessListener(entityIdRegistry()); 29 | } 30 | 31 | @Bean 32 | public ItemCountingItemProcessListener ItemCountingItemProcessListener() { 33 | return new ItemCountingItemProcessListener(itemsCounter()); 34 | } 35 | 36 | @Bean 37 | public CacheInversionStepExecutionListener cacheInversionStepExecutionListener() { 38 | return new CacheInversionStepExecutionListener(entityIdRegistry()); 39 | } 40 | 41 | @Bean 42 | public StopWatchStepExecutionListener stopWatchStepExecutionListener() { 43 | return new StopWatchStepExecutionListener(itemsCounter()); 44 | } 45 | 46 | @Bean 47 | public StringNormalizingItemReadListener stringNormalizingItemReadListener() { 48 | return new StringNormalizingItemReadListener(); 49 | } 50 | 51 | @Bean 52 | public ExitSignalJobExecutionListener exitSignalJobExecutionListener(CountDownLatch exitLatch) { 53 | return new ExitSignalJobExecutionListener(exitLatch); 54 | } 55 | 56 | @Bean 57 | public IdCachingJobExecutionListener idCachingJobExecutionListener(DSLContext context) { 58 | return new IdCachingJobExecutionListener(entityIdRegistry(), context); 59 | } 60 | 61 | @Bean 62 | public ClearanceJobExecutionListener clearanceJobExecutionListener(FileUtil fileUtil) { 63 | return new ClearanceJobExecutionListener(entityIdRegistry(), fileUtil); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/LabelSubItemsProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.LABEL; 4 | 5 | import io.dsub.discogs.batch.domain.label.LabelSubItemsXML; 6 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 7 | import io.dsub.discogs.batch.util.ReflectionUtil; 8 | import io.dsub.discogs.jooq.tables.records.LabelUrlRecord; 9 | import java.time.Clock; 10 | import java.time.LocalDateTime; 11 | import java.util.ArrayList; 12 | import java.util.Collection; 13 | import java.util.List; 14 | import lombok.RequiredArgsConstructor; 15 | import org.jooq.UpdatableRecord; 16 | import org.springframework.batch.item.ItemProcessor; 17 | 18 | @RequiredArgsConstructor 19 | public class LabelSubItemsProcessor 20 | implements ItemProcessor>> { 21 | 22 | private final EntityIdRegistry idRegistry; 23 | 24 | @Override 25 | public Collection> process(LabelSubItemsXML item) { 26 | if (item.getId() == null || item.getId() < 1) { 27 | return null; 28 | } 29 | 30 | ReflectionUtil.normalizeStringFields(item); 31 | 32 | List> records = new ArrayList<>(); 33 | 34 | Integer labelId = item.getId(); 35 | 36 | if (item.getLabelSubLabels() != null) { 37 | item.getLabelSubLabels().stream() 38 | .filter(subLabel -> isExistingLabel(subLabel.getSubLabelId())) 39 | .map(xml -> xml.getRecord(labelId)) 40 | .forEach(records::add); 41 | } 42 | 43 | if (item.getUrls() != null) { 44 | item.getUrls().stream() 45 | .filter(url -> !url.isBlank()) 46 | .distinct() 47 | .map(url -> getLabelUrlRecord(labelId, url)) 48 | .forEach(records::add); 49 | } 50 | 51 | return records; 52 | } 53 | 54 | private LabelUrlRecord getLabelUrlRecord(Integer labelId, String url) { 55 | return new LabelUrlRecord() 56 | .setLabelId(labelId) 57 | .setUrl(url) 58 | .setHash(url.hashCode()) 59 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())) 60 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())); 61 | } 62 | 63 | private boolean isExistingLabel(Integer labelId) { 64 | return idRegistry.exists(LABEL, labelId); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/DefaultJobParameterResolver.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import io.dsub.discogs.batch.argument.ArgType; 4 | import io.dsub.discogs.batch.config.BatchConfig; 5 | import io.dsub.discogs.batch.dump.DumpDependencyResolver; 6 | import io.dsub.discogs.batch.exception.DumpNotFoundException; 7 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 8 | import java.util.Properties; 9 | import lombok.RequiredArgsConstructor; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.boot.ApplicationArguments; 12 | import org.springframework.stereotype.Component; 13 | 14 | @Slf4j 15 | @Component 16 | @RequiredArgsConstructor 17 | public class DefaultJobParameterResolver implements JobParameterResolver { 18 | 19 | private static final String CHUNK_SIZE = ArgType.CHUNK_SIZE.getGlobalName(); 20 | private static final String STRICT = ArgType.STRICT.getGlobalName(); 21 | 22 | private final DumpDependencyResolver dumpDependencyResolver; 23 | 24 | @Override 25 | public Properties resolve(ApplicationArguments args) 26 | throws InvalidArgumentException, DumpNotFoundException { 27 | Properties props = new Properties(); 28 | dumpDependencyResolver 29 | .resolve(args) 30 | .forEach(dump -> props.put(dump.getType().toString(), dump.getETag())); // add all dumps 31 | props.put(CHUNK_SIZE, String.valueOf(parseChunkSize(args))); 32 | 33 | if (args.containsOption(STRICT)) { 34 | props.put(STRICT, "true"); 35 | } 36 | return props; 37 | } 38 | 39 | protected int parseChunkSize(ApplicationArguments args) throws InvalidArgumentException { 40 | String chunkSizeOptName = ArgType.CHUNK_SIZE.getGlobalName(); 41 | if (args.containsOption(chunkSizeOptName)) { 42 | String toParse = args.getOptionValues(chunkSizeOptName).get(0); 43 | try { 44 | log.debug("found entry for " + chunkSizeOptName + ": " + toParse); 45 | return Integer.parseInt(toParse); 46 | } catch (NumberFormatException ignored) { 47 | throw new InvalidArgumentException("failed to parse " + chunkSizeOptName + ": " + toParse); 48 | } 49 | } 50 | log.debug( 51 | chunkSizeOptName 52 | + " not specified. returning default value: " 53 | + BatchConfig.DEFAULT_CHUNK_SIZE); 54 | return BatchConfig.DEFAULT_CHUNK_SIZE; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/validator/TypeArgumentValidatorTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | 5 | import java.util.List; 6 | import java.util.stream.Collectors; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | import org.springframework.boot.ApplicationArguments; 10 | import org.springframework.boot.DefaultApplicationArguments; 11 | 12 | class TypeArgumentValidatorTest { 13 | 14 | final TypeArgumentValidator validator = new TypeArgumentValidator(); 15 | 16 | @BeforeEach 17 | void setUp() { 18 | } 19 | 20 | @Test 21 | void whenDuplicatedTypeArgExists__ThenShouldIncludeAllOfThemInReport() { 22 | 23 | ApplicationArguments args = new DefaultApplicationArguments("--types=hello", "--type=hi"); 24 | // when 25 | ValidationResult result = validator.validate(args); 26 | 27 | // then 28 | assertThat(result.getIssues().size()).isEqualTo(1); 29 | assertThat(result.getIssues().get(0)).contains("types", "type"); 30 | } 31 | 32 | @Test 33 | void whenMalformedTypeValueExists__ThenShouldReportEachOfThemInReport() { 34 | List values = List.of("--types=hello", "--types=world", "--types=malformed"); 35 | ApplicationArguments args = new DefaultApplicationArguments(values.toArray(String[]::new)); 36 | 37 | // when 38 | ValidationResult result = validator.validate(args); 39 | 40 | // then 41 | assertThat(result.getIssues().size()).isEqualTo(3); 42 | for (String value : values) { 43 | assertThat("unknown type argument value: " + value.replaceAll("--types=", "")) 44 | .isIn(result.getIssues()); 45 | } 46 | } 47 | 48 | @Test 49 | void whenProperTypeValuesPresented__ThenShouldNotReportAnyIssue() { 50 | String prefix = "--type="; 51 | List wantedTypes = List.of("release", "artist", "master", "label"); 52 | String[] optionArgs = 53 | wantedTypes.stream() 54 | .map(value -> prefix + value) 55 | .collect(Collectors.toList()) 56 | .toArray(String[]::new); 57 | 58 | ApplicationArguments args = new DefaultApplicationArguments(optionArgs); 59 | 60 | // when 61 | ValidationResult result = validator.validate(args); 62 | 63 | // then 64 | assertThat(result.getIssues().size()).isEqualTo(0); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/registry/DefaultEntityIdRegistry.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.registry; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.ConcurrentSkipListSet; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | 8 | @Slf4j 9 | public class DefaultEntityIdRegistry implements EntityIdRegistry { 10 | 11 | private final IdCache artistCache = new IdCache(Type.ARTIST); 12 | private final IdCache masterCache = new IdCache(Type.MASTER); 13 | private final IdCache labelCache = new IdCache(Type.LABEL); 14 | private final IdCache releaseItemCache = new IdCache(Type.LABEL); 15 | 16 | private final ConcurrentSkipListSet genreSet = new ConcurrentSkipListSet<>(); 17 | private final ConcurrentSkipListSet styleSet = new ConcurrentSkipListSet<>(); 18 | 19 | @Override 20 | public boolean exists(Type type, Integer id) { 21 | if (id == null || id < 1) { 22 | return false; 23 | } 24 | return getLongIdCache(type).exists(id); 25 | } 26 | 27 | @Override 28 | public boolean exists(Type type, String id) { 29 | return getStringIdSetByType(type).contains(id); 30 | } 31 | 32 | @Override 33 | public void put(Type type, Integer id) { 34 | if (type != null && id != null) { 35 | getLongIdCache(type).add(id); 36 | } 37 | } 38 | 39 | @Override 40 | public void put(Type type, String id) { 41 | if (id != null && !id.isBlank()) { 42 | getStringIdSetByType(type).add(id); 43 | } 44 | } 45 | 46 | @Override 47 | public void invert(Type type) { 48 | switch (type) { 49 | case ARTIST -> artistCache.invert(); 50 | case LABEL -> labelCache.invert(); 51 | case MASTER -> masterCache.invert(); 52 | } 53 | } 54 | 55 | @Override 56 | public void clearAll() { 57 | for (Type t : List.of(Type.ARTIST, Type.LABEL, Type.MASTER, Type.RELEASE)) { 58 | getLongIdCache(t).getConcurrentSkipListSet().clear(); 59 | } 60 | genreSet.clear(); 61 | styleSet.clear(); 62 | } 63 | 64 | @Override 65 | public ConcurrentSkipListSet getStringIdSetByType(Type type) { 66 | if (type.equals(Type.GENRE)) { 67 | return genreSet; 68 | } 69 | return styleSet; 70 | } 71 | 72 | @Override 73 | public IdCache getLongIdCache(Type type) { 74 | return switch (type) { 75 | case ARTIST -> artistCache; 76 | case LABEL -> labelCache; 77 | case MASTER -> masterCache; 78 | default -> releaseItemCache; 79 | }; 80 | } 81 | 82 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/ReleaseItemCoreProcessor.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.release.ReleaseItemXML; 4 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 5 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 6 | import io.dsub.discogs.batch.util.DefaultMalformedDateParser; 7 | import io.dsub.discogs.batch.util.MalformedDateParser; 8 | import io.dsub.discogs.batch.util.ReflectionUtil; 9 | import io.dsub.discogs.jooq.tables.records.ReleaseItemRecord; 10 | import java.time.Clock; 11 | import java.time.LocalDateTime; 12 | import lombok.RequiredArgsConstructor; 13 | import org.springframework.batch.item.ItemProcessor; 14 | 15 | @RequiredArgsConstructor 16 | public class ReleaseItemCoreProcessor implements ItemProcessor { 17 | 18 | private final MalformedDateParser parser = new DefaultMalformedDateParser(); 19 | private final EntityIdRegistry idRegistry; 20 | 21 | @Override 22 | public ReleaseItemRecord process(ReleaseItemXML release) throws Exception { 23 | 24 | if (release.getId() == null || release.getId() < 1) { 25 | return null; 26 | } 27 | 28 | ReflectionUtil.normalizeStringFields(release); 29 | 30 | Integer masterId = null; 31 | 32 | if (release.getMaster() != null && release.getMaster().getMasterId() != null) { 33 | Integer id = release.getMaster().getMasterId(); 34 | if (idRegistry.exists(DefaultEntityIdRegistry.Type.MASTER, id)) { 35 | masterId = id; 36 | } 37 | } 38 | 39 | return new ReleaseItemRecord() 40 | .setId(release.getId()) 41 | .setTitle(release.getTitle()) 42 | .setStatus(release.getStatus()) 43 | .setCountry(release.getCountry()) 44 | .setDataQuality(release.getDataQuality()) 45 | .setReleaseDate(parser.parse(release.getReleaseDate())) 46 | .setHasValidDay(parser.isDayValid(release.getReleaseDate())) 47 | .setHasValidMonth(parser.isMonthValid(release.getReleaseDate())) 48 | .setHasValidYear(parser.isYearValid(release.getReleaseDate())) 49 | .setListedReleaseDate(release.getReleaseDate()) 50 | .setIsMaster(release.getMaster() != null && release.getMaster().isMaster()) 51 | .setMasterId(masterId) 52 | .setNotes(release.getNotes()) 53 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 54 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/CacheInversionStepExecutionListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 4 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 5 | import io.dsub.discogs.batch.job.step.core.ArtistStepConfig; 6 | import io.dsub.discogs.batch.job.step.core.LabelStepConfig; 7 | import io.dsub.discogs.batch.job.step.core.MasterStepConfig; 8 | import io.dsub.discogs.batch.job.step.core.ReleaseItemStepConfig; 9 | import lombok.RequiredArgsConstructor; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.batch.core.ExitStatus; 12 | import org.springframework.batch.core.StepExecution; 13 | import org.springframework.batch.core.StepExecutionListener; 14 | 15 | @Slf4j 16 | @RequiredArgsConstructor 17 | public class CacheInversionStepExecutionListener implements StepExecutionListener { 18 | 19 | private static final String INVERT_CACHE_MSG = "inverting {} id cache"; 20 | private static final String ARTIST = "artist"; 21 | private static final String LABEL = "label"; 22 | private static final String MASTER = "master"; 23 | private static final String RELEASE_ITEM = "release item"; 24 | private final EntityIdRegistry idRegistry; 25 | 26 | @Override 27 | public void beforeStep(StepExecution stepExecution) { 28 | } 29 | 30 | @Override 31 | public ExitStatus afterStep(StepExecution stepExecution) { 32 | boolean doMaster = stepExecution.getJobParameters().getParameters().containsKey(MASTER); 33 | boolean doRelease = stepExecution.getJobParameters().getParameters().containsKey(RELEASE_ITEM); 34 | 35 | // current 36 | String stepName = stepExecution.getStepName(); 37 | 38 | if (stepName.equals(ArtistStepConfig.ARTIST_CORE_INSERTION_STEP) && (doMaster || doRelease)) { 39 | log.info(INVERT_CACHE_MSG, ARTIST); 40 | idRegistry.invert(DefaultEntityIdRegistry.Type.ARTIST); 41 | } 42 | 43 | if (stepName.equals(LabelStepConfig.LABEL_CORE_INSERTION_STEP) && (doMaster || doRelease)) { 44 | log.info(INVERT_CACHE_MSG, LABEL); 45 | idRegistry.invert(DefaultEntityIdRegistry.Type.LABEL); 46 | } 47 | 48 | if (stepName.equals(MasterStepConfig.MASTER_CORE_INSERTION_STEP) && doRelease) { 49 | log.info(INVERT_CACHE_MSG, MASTER); 50 | idRegistry.invert(DefaultEntityIdRegistry.Type.MASTER); 51 | } 52 | 53 | if (stepName.equals(ReleaseItemStepConfig.RELEASE_ITEM_CORE_INSERTION_STEP) && (doMaster)) { 54 | log.info(INVERT_CACHE_MSG, RELEASE_ITEM); 55 | idRegistry.invert(DefaultEntityIdRegistry.Type.RELEASE); 56 | } 57 | 58 | return stepExecution.getExitStatus(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/JobPreparationRunner.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import com.zaxxer.hikari.HikariDataSource; 4 | import io.dsub.discogs.batch.exception.DumpNotFoundException; 5 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 6 | import io.dsub.discogs.batch.job.JobParameterResolver; 7 | 8 | import java.io.BufferedReader; 9 | import java.sql.Connection; 10 | import java.sql.PreparedStatement; 11 | import java.util.ArrayList; 12 | import java.util.Arrays; 13 | import java.util.List; 14 | import java.util.Properties; 15 | import java.util.concurrent.CountDownLatch; 16 | import java.util.stream.Collectors; 17 | import javax.sql.DataSource; 18 | 19 | import lombok.RequiredArgsConstructor; 20 | import lombok.extern.slf4j.Slf4j; 21 | import org.jline.utils.InputStreamReader; 22 | import org.springframework.batch.core.JobParameters; 23 | import org.springframework.batch.core.converter.JobParametersConverter; 24 | import org.springframework.boot.ApplicationArguments; 25 | import org.springframework.boot.ApplicationRunner; 26 | import org.springframework.context.annotation.Bean; 27 | import org.springframework.context.annotation.Configuration; 28 | import org.springframework.core.annotation.Order; 29 | import org.springframework.core.io.DefaultResourceLoader; 30 | import org.springframework.core.io.Resource; 31 | import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; 32 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 33 | 34 | @Slf4j 35 | @Order(0) 36 | @Configuration 37 | @RequiredArgsConstructor 38 | public class JobPreparationRunner implements ApplicationRunner { 39 | 40 | private final JobParameterResolver jobParameterResolver; 41 | private final JobParametersConverter jobParametersConverter; 42 | private final DataSource dataSource; 43 | private final ThreadPoolTaskExecutor taskExecutor; 44 | 45 | @Override 46 | public void run(ApplicationArguments args) { 47 | int poolSize = taskExecutor.getMaxPoolSize() + 3; 48 | 49 | if (dataSource instanceof HikariDataSource) { 50 | log.info("setting db connection pool size to " + poolSize); 51 | ((HikariDataSource) dataSource).setMaximumPoolSize(poolSize); 52 | } 53 | } 54 | 55 | @Bean 56 | public CountDownLatch exitLatch() { 57 | return new CountDownLatch(1); 58 | } 59 | 60 | @Bean(name = "discogsJobParameters") 61 | public JobParameters getDiscogsJobParameters(ApplicationArguments args) 62 | throws InvalidArgumentException, DumpNotFoundException { 63 | log.info("resolving given job parameters"); 64 | Properties props = jobParameterResolver.resolve(args); 65 | return jobParametersConverter.getJobParameters(props); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/StopWatchStepExecutionListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.batch.core.BatchStatus; 6 | import org.springframework.batch.core.ExitStatus; 7 | import org.springframework.batch.core.StepExecution; 8 | import org.springframework.batch.core.StepExecutionListener; 9 | import org.springframework.util.NumberUtils; 10 | import org.springframework.util.StopWatch; 11 | 12 | @Slf4j 13 | public class StopWatchStepExecutionListener implements StepExecutionListener { 14 | 15 | private final AtomicLong itemsCounter; 16 | private StopWatch stopWatch; 17 | 18 | public StopWatchStepExecutionListener(final AtomicLong itemsCounter) { 19 | this.itemsCounter = itemsCounter; 20 | this.stopWatch = new StopWatch(); 21 | this.stopWatch.setKeepTaskList(false); 22 | this.init(); 23 | } 24 | 25 | private void init() { 26 | this.stopWatch = getStopWatch(); 27 | this.itemsCounter.set(0); 28 | } 29 | 30 | @Override 31 | public void beforeStep(StepExecution stepExecution) { 32 | stepExecution.setStatus(BatchStatus.STARTING); 33 | getStopWatch().start(stepExecution.getStepName()); 34 | } 35 | 36 | @Override 37 | public ExitStatus afterStep(StepExecution stepExecution) { 38 | printStepDetails(itemsCounter.get() == 0 ? stepExecution.getWriteCount() : itemsCounter.get()); 39 | stepExecution.setStatus(BatchStatus.COMPLETED); 40 | this.init(); 41 | return ExitStatus.COMPLETED; 42 | } 43 | 44 | /** 45 | * reports step execution details as log. 46 | * 47 | * @param writeCount items has been written during step 48 | */ 49 | private void printStepDetails(long writeCount) { 50 | int seconds = getTotalTimeSeconds(); 51 | 52 | if (seconds == 0) { // nothing to report... 53 | return; 54 | } 55 | 56 | long itemsPerSecond = writeCount / seconds; 57 | 58 | String timeTookSeconds = String.valueOf(seconds); 59 | String itemsProcPerSec = itemsPerSecond + "/s"; 60 | String taskName = getStopWatch().getLastTaskName(); 61 | 62 | log.info( 63 | "task {} took {} seconds and updated {} items. processed items per second: {}", 64 | taskName, 65 | timeTookSeconds, 66 | writeCount, 67 | itemsProcPerSec); 68 | } 69 | 70 | protected StopWatch getStopWatch() { 71 | return stopWatch; 72 | } 73 | 74 | protected int getTotalTimeSeconds() { 75 | StopWatch stopWatch = getStopWatch(); 76 | if (stopWatch.isRunning()) { 77 | stopWatch.stop(); 78 | } 79 | return NumberUtils.convertNumberToTargetClass(stopWatch.getTotalTimeSeconds(), Integer.class); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/config/DiscogsBatchConfigurer.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.config; 2 | 3 | import javax.sql.DataSource; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.batch.core.configuration.annotation.BatchConfigurer; 6 | import org.springframework.batch.core.explore.JobExplorer; 7 | import org.springframework.batch.core.explore.support.JobExplorerFactoryBean; 8 | import org.springframework.batch.core.launch.JobLauncher; 9 | import org.springframework.batch.core.launch.support.SimpleJobLauncher; 10 | import org.springframework.batch.core.repository.JobRepository; 11 | import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; 12 | import org.springframework.jdbc.datasource.DataSourceTransactionManager; 13 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 14 | import org.springframework.stereotype.Component; 15 | import org.springframework.transaction.PlatformTransactionManager; 16 | 17 | /** 18 | * Default batch configuration implementation of {@link BatchConfigurer}. 19 | */ 20 | @Component 21 | @RequiredArgsConstructor 22 | public class DiscogsBatchConfigurer implements BatchConfigurer { 23 | 24 | private final DataSource dataSource; 25 | private final ThreadPoolTaskExecutor taskExecutor; 26 | 27 | @Override 28 | public JobRepository getJobRepository() throws Exception { 29 | JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean(); 30 | factory.setTransactionManager(getTransactionManager()); 31 | factory.setDataSource(dataSource); 32 | factory.afterPropertiesSet(); 33 | return factory.getObject(); 34 | } 35 | 36 | /** 37 | * {@link PlatformTransactionManager} to be used for spring batch context. 38 | * 39 | * @return a new, separated transaction manager from the business logics. 40 | */ 41 | @Override 42 | public PlatformTransactionManager getTransactionManager() { 43 | DataSourceTransactionManager transactionManager = new DataSourceTransactionManager(dataSource); 44 | transactionManager.setDataSource(dataSource); 45 | transactionManager.afterPropertiesSet(); 46 | return transactionManager; 47 | } 48 | 49 | @Override 50 | public JobLauncher getJobLauncher() throws Exception { 51 | SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); 52 | jobLauncher.setTaskExecutor(taskExecutor); 53 | jobLauncher.setJobRepository(getJobRepository()); 54 | jobLauncher.afterPropertiesSet(); 55 | return jobLauncher; 56 | } 57 | 58 | @Override 59 | public JobExplorer getJobExplorer() throws Exception { 60 | JobExplorerFactoryBean factory = new JobExplorerFactoryBean(); 61 | factory.setDataSource(dataSource); 62 | factory.afterPropertiesSet(); 63 | return factory.getObject(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/reader/DumpItemReaderBuilderTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.reader; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.assertj.core.api.Assertions.catchThrowable; 5 | import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 6 | import static org.junit.jupiter.api.Assertions.fail; 7 | import static org.mockito.Mockito.when; 8 | 9 | import io.dsub.discogs.batch.domain.artist.ArtistSubItemsXML; 10 | import io.dsub.discogs.batch.dump.DiscogsDump; 11 | import io.dsub.discogs.batch.dump.EntityType; 12 | import io.dsub.discogs.batch.exception.FileException; 13 | import io.dsub.discogs.batch.util.FileUtil; 14 | import java.nio.file.Path; 15 | import org.junit.jupiter.api.BeforeEach; 16 | import org.junit.jupiter.api.Test; 17 | import org.mockito.Mockito; 18 | 19 | class DumpItemReaderBuilderTest { 20 | 21 | DiscogsDump dump; 22 | FileUtil fileUtil; 23 | DiscogsDumpItemReaderBuilder readerBuilder; 24 | 25 | @BeforeEach 26 | void setUp() { 27 | fileUtil = Mockito.mock(FileUtil.class); 28 | dump = Mockito.mock(DiscogsDump.class); 29 | readerBuilder = new DiscogsDumpItemReaderBuilder(fileUtil); 30 | } 31 | 32 | @Test 33 | void whenBuild__ShouldNotThrow() { 34 | try { 35 | when(fileUtil.getFilePath("artist.xml.gz")) 36 | .thenReturn(Path.of("src/test/resources/test/reader/artist.xml.gz")); 37 | when(dump.getFileName()).thenReturn("artist.xml.gz"); 38 | when(dump.getType()).thenReturn(EntityType.ARTIST); 39 | assertDoesNotThrow(() -> readerBuilder.build(ArtistSubItemsXML.class, dump)); 40 | } catch (FileException e) { 41 | fail(e); 42 | } 43 | } 44 | 45 | @Test 46 | void whenTypeNotSet__ShouldThrow() { 47 | try { 48 | when(dump.getType()).thenReturn(null); 49 | when(dump.getFileName()).thenReturn("src/test/resources/test/reader/artist.xml.gz"); 50 | when(fileUtil.getFilePath(dump.getFileName())) 51 | .thenReturn(Path.of("src/test/resources/test/reader/artist.xml.gz")); 52 | Throwable t = catchThrowable(() -> readerBuilder.build(ArtistSubItemsXML.class, dump)); 53 | assertThat(t).hasMessageContaining("type of DiscogsDump cannot be null"); 54 | } catch (FileException e) { 55 | fail(e); 56 | } 57 | } 58 | 59 | @Test 60 | void whenUriNotSet__ShouldThrow() { 61 | try { 62 | when(dump.getFileName()).thenReturn(null); 63 | when(fileUtil.getFilePath(dump.getFileName())).thenThrow(FileException.class); 64 | Throwable t = catchThrowable(() -> readerBuilder.build(ArtistSubItemsXML.class, dump)); 65 | assertThat(t).hasMessageContaining("fileName of DiscogsDump cannot be null"); 66 | } catch (FileException e) { 67 | fail(); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/BatchInfrastructureConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.mockito.Mockito.mock; 5 | 6 | import ch.qos.logback.classic.Level; 7 | import io.dsub.discogs.batch.dump.service.DiscogsDumpService; 8 | import io.dsub.discogs.batch.testutil.LogSpy; 9 | import io.dsub.discogs.batch.util.FileUtil; 10 | import java.util.concurrent.CountDownLatch; 11 | import org.jooq.DSLContext; 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | import org.junit.jupiter.api.extension.RegisterExtension; 15 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 16 | import org.springframework.batch.core.repository.JobRepository; 17 | import org.springframework.boot.DefaultApplicationArguments; 18 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 19 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 20 | 21 | public class BatchInfrastructureConfigTest { 22 | 23 | ApplicationContextRunner ctx; 24 | 25 | @RegisterExtension 26 | LogSpy logSpy = new LogSpy(); 27 | 28 | @BeforeEach 29 | void setUp() { 30 | ctx = new ApplicationContextRunner() 31 | .withBean(DSLContext.class, () -> mock(DSLContext.class)) 32 | .withBean(JobRepository.class, () -> mock(JobRepository.class)) 33 | .withBean(CountDownLatch.class, () -> mock(CountDownLatch.class)) 34 | .withBean(DiscogsDumpService.class, () -> mock(DiscogsDumpService.class)) 35 | .withBean(StepBuilderFactory.class, () -> mock(StepBuilderFactory.class)) 36 | .withBean(ThreadPoolTaskExecutor.class, () -> mock(ThreadPoolTaskExecutor.class)) 37 | .withUserConfiguration(BatchInfrastructureConfig.class); 38 | } 39 | 40 | @Test 41 | void givenMountOption__ShouldSetNotBeingTemporaryFile() { 42 | // given 43 | ctx = ctx.withBean(DefaultApplicationArguments.class, "--mount"); 44 | 45 | // when 46 | ctx.run(it -> assertThat(it).hasSingleBean(FileUtil.class)); 47 | 48 | // then 49 | assertThat(logSpy.getLogsByExactLevelAsString(Level.INFO, true)) 50 | .hasSize(1) 51 | .first() 52 | .isEqualTo("detected mount option. keeping file..."); 53 | } 54 | 55 | @Test 56 | void givenOptionWithoutMount__ShouldSetAsTemporaryFile() { 57 | // given 58 | ctx = ctx.withBean(DefaultApplicationArguments.class); 59 | 60 | // when 61 | ctx.run(it -> assertThat(it).hasSingleBean(FileUtil.class)); 62 | 63 | // then 64 | assertThat(logSpy.getLogsByExactLevelAsString(Level.INFO, true)) 65 | .hasSize(1) 66 | .first() 67 | .isEqualTo("mount option not set. files will be removed after the job."); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/BatchInfrastructureConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job; 2 | 3 | import io.dsub.discogs.batch.argument.ArgType; 4 | import io.dsub.discogs.batch.dump.DiscogsDump; 5 | import io.dsub.discogs.batch.dump.EntityType; 6 | import io.dsub.discogs.batch.job.decider.MasterMainReleaseStepJobExecutionDecider; 7 | import io.dsub.discogs.batch.job.listener.BatchListenerConfig; 8 | import io.dsub.discogs.batch.job.processor.ItemProcessorConfig; 9 | import io.dsub.discogs.batch.job.reader.DiscogsDumpItemReaderBuilder; 10 | import io.dsub.discogs.batch.job.reader.ItemReaderConfig; 11 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 12 | import io.dsub.discogs.batch.job.step.GlobalStepConfig; 13 | import io.dsub.discogs.batch.job.tasklet.GenreStyleInsertionTasklet; 14 | import io.dsub.discogs.batch.job.writer.ItemWriterConfig; 15 | import io.dsub.discogs.batch.util.FileUtil; 16 | import io.dsub.discogs.batch.util.SimpleFileUtil; 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | import lombok.extern.slf4j.Slf4j; 20 | import org.springframework.beans.factory.annotation.Autowired; 21 | import org.springframework.boot.ApplicationArguments; 22 | import org.springframework.context.annotation.Bean; 23 | import org.springframework.context.annotation.Configuration; 24 | import org.springframework.context.annotation.Import; 25 | 26 | @Slf4j 27 | @Configuration 28 | @Import( 29 | value = { 30 | GlobalStepConfig.class, 31 | ItemReaderConfig.class, 32 | ItemProcessorConfig.class, 33 | ItemWriterConfig.class, 34 | BatchListenerConfig.class, 35 | GenreStyleInsertionTasklet.class 36 | }) 37 | public class BatchInfrastructureConfig { 38 | 39 | private ApplicationArguments args; 40 | 41 | @Autowired 42 | public void setArgs(ApplicationArguments args) { 43 | this.args = args; 44 | } 45 | 46 | @Bean 47 | public Map dumpMap() { 48 | return new HashMap<>(); 49 | } 50 | 51 | @Bean 52 | public MasterMainReleaseStepJobExecutionDecider masterMainReleaseStepJobExecutionDecider( 53 | DefaultEntityIdRegistry registry) { 54 | return new MasterMainReleaseStepJobExecutionDecider(registry); 55 | } 56 | 57 | // TODO: test! 58 | @Bean 59 | public FileUtil fileUtil() { 60 | boolean keepFile = args.containsOption(ArgType.MOUNT.getGlobalName()); 61 | FileUtil fileUtil = SimpleFileUtil.builder().isTemporary(!keepFile).build(); 62 | if (keepFile) { 63 | log.info("detected mount option. keeping file..."); 64 | } else { 65 | log.info("mount option not set. files will be removed after the job."); 66 | } 67 | return fileUtil; 68 | } 69 | 70 | @Bean 71 | public DiscogsDumpItemReaderBuilder discogsDumpItemReaderBuilder() { 72 | return new DiscogsDumpItemReaderBuilder(fileUtil()); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/registry/IdCache.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.registry; 2 | 3 | import java.util.Objects; 4 | import java.util.OptionalInt; 5 | import java.util.concurrent.ConcurrentSkipListSet; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | import java.util.stream.IntStream; 8 | import lombok.Getter; 9 | 10 | public class IdCache { 11 | 12 | @Getter 13 | private final DefaultEntityIdRegistry.Type type; 14 | @Getter 15 | private final ConcurrentSkipListSet concurrentSkipListSet; 16 | private boolean inverted = false; 17 | private AtomicInteger lastMax = null; 18 | 19 | public IdCache(DefaultEntityIdRegistry.Type type) { 20 | this.type = type; 21 | this.concurrentSkipListSet = new ConcurrentSkipListSet<>(); 22 | } 23 | 24 | public boolean exists(Integer item) { 25 | if (item == null || lastMax == null) { 26 | return false; 27 | } 28 | 29 | if (item > lastMax.get()) { 30 | return false; 31 | } 32 | 33 | if (inverted) { 34 | return !concurrentSkipListSet.contains(item); 35 | } 36 | return concurrentSkipListSet.contains(item); 37 | } 38 | 39 | public void add(Integer item) { 40 | if (item == null) { 41 | return; 42 | } 43 | if (lastMax == null) { 44 | lastMax = new AtomicInteger(item); 45 | } else if (lastMax.get() < item) { 46 | lastMax.set(item); 47 | } 48 | if (inverted) { 49 | return; 50 | } 51 | this.concurrentSkipListSet.add(item); 52 | } 53 | 54 | public boolean isInverted() { 55 | return this.inverted; 56 | } 57 | 58 | public void invert() { 59 | if (!this.inverted) { 60 | doInvertFromNonInverted(); 61 | } else { 62 | doInvertFromInverted(); 63 | } 64 | this.inverted = !inverted; 65 | System.gc(); // force gc call for mem clear 66 | } 67 | 68 | private void doInvertFromInverted() { 69 | if (lastMax == null || lastMax.get() < 1) { 70 | return; 71 | } 72 | flip(); 73 | } 74 | 75 | private void doInvertFromNonInverted() { 76 | OptionalInt optMax = 77 | this.concurrentSkipListSet.stream().filter(Objects::nonNull).mapToInt(num -> num).max(); 78 | 79 | if (optMax.isEmpty()) { 80 | return; 81 | } 82 | 83 | int max = optMax.getAsInt(); 84 | 85 | if (lastMax == null) { 86 | lastMax = new AtomicInteger(max); 87 | } else if (lastMax.get() < max) { 88 | lastMax.set(max); 89 | } 90 | 91 | flip(); 92 | } 93 | 94 | private void flip() { 95 | IntStream.range(1, lastMax.get() + 1).forEach(this::flipSingleValue); 96 | } 97 | 98 | private void flipSingleValue(int intValue) { 99 | if (this.concurrentSkipListSet.contains(intValue)) { 100 | this.concurrentSkipListSet.remove(intValue); 101 | return; 102 | } 103 | this.concurrentSkipListSet.add(intValue); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/listener/IdCachingItemProcessListener.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.ARTIST; 4 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.GENRE; 5 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.LABEL; 6 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.MASTER; 7 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.RELEASE; 8 | import static io.dsub.discogs.batch.job.registry.EntityIdRegistry.Type.STYLE; 9 | 10 | import io.dsub.discogs.batch.domain.artist.ArtistXML; 11 | import io.dsub.discogs.batch.domain.label.LabelXML; 12 | import io.dsub.discogs.batch.domain.master.MasterXML; 13 | import io.dsub.discogs.batch.domain.release.ReleaseItemXML; 14 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 15 | import io.dsub.discogs.batch.job.registry.EntityIdRegistry; 16 | import java.util.List; 17 | import java.util.Objects; 18 | import lombok.RequiredArgsConstructor; 19 | import org.springframework.batch.core.ItemProcessListener; 20 | 21 | @RequiredArgsConstructor 22 | public class IdCachingItemProcessListener implements ItemProcessListener { 23 | 24 | private final EntityIdRegistry idRegistry; 25 | 26 | /* No Op */ 27 | @Override 28 | public void beforeProcess(Object item) { 29 | 30 | } 31 | 32 | @Override 33 | public void afterProcess(Object pulled, Object result) { 34 | if (result == null) { 35 | return; 36 | } 37 | if (pulled instanceof ArtistXML artist) { 38 | if (artist.getId() != null) { 39 | idRegistry.put(ARTIST, artist.getId()); 40 | } 41 | } else if (pulled instanceof LabelXML label) { 42 | if (label.getId() != null) { 43 | idRegistry.put(LABEL, label.getId()); 44 | } 45 | } else if (pulled instanceof MasterXML master) { 46 | if (master.getId() != null) { 47 | idRegistry.put(MASTER, master.getId()); 48 | cacheStringTypedItems(STYLE, master.getStyles()); 49 | cacheStringTypedItems(GENRE, master.getGenres()); 50 | } 51 | } else if (pulled instanceof ReleaseItemXML releaseItem) { 52 | if (releaseItem.getId() != null) { 53 | idRegistry.put(RELEASE, releaseItem.getId()); 54 | cacheStringTypedItems(GENRE, releaseItem.getGenres()); 55 | cacheStringTypedItems(STYLE, releaseItem.getStyles()); 56 | } 57 | } 58 | } 59 | 60 | private void cacheStringTypedItems(DefaultEntityIdRegistry.Type type, List values) { 61 | if (values == null || values.isEmpty()) { 62 | return; 63 | } 64 | values.stream() 65 | .filter(Objects::nonNull) 66 | .map(String::trim) 67 | .filter(val -> !val.isBlank()) 68 | .forEach(value -> idRegistry.put(type, value)); 69 | } 70 | 71 | /* No Op */ 72 | @Override 73 | public void onProcessError(Object item, Exception e) { 74 | 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/master/MasterSubItemsXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.master; 2 | 3 | import io.dsub.discogs.batch.domain.HashXML; 4 | import io.dsub.discogs.batch.domain.SubItemXML; 5 | import io.dsub.discogs.jooq.tables.records.MasterArtistRecord; 6 | import io.dsub.discogs.jooq.tables.records.MasterVideoRecord; 7 | import java.time.Clock; 8 | import java.time.LocalDateTime; 9 | import java.util.List; 10 | import javax.xml.bind.annotation.XmlAccessType; 11 | import javax.xml.bind.annotation.XmlAccessorType; 12 | import javax.xml.bind.annotation.XmlAttribute; 13 | import javax.xml.bind.annotation.XmlElement; 14 | import javax.xml.bind.annotation.XmlElementWrapper; 15 | import javax.xml.bind.annotation.XmlRootElement; 16 | import lombok.Data; 17 | import lombok.EqualsAndHashCode; 18 | 19 | @Data 20 | @XmlRootElement(name = "master") 21 | @XmlAccessorType(XmlAccessType.FIELD) 22 | @EqualsAndHashCode(callSuper = false) 23 | public class MasterSubItemsXML { 24 | 25 | @XmlAttribute(name = "id") 26 | private Integer id; 27 | 28 | @XmlElementWrapper(name = "artists") 29 | @XmlElement(name = "artist") 30 | private List masterArtists; 31 | 32 | @XmlElementWrapper(name = "genres") 33 | @XmlElement(name = "genre") 34 | private List genres; 35 | 36 | @XmlElementWrapper(name = "styles") 37 | @XmlElement(name = "style") 38 | private List styles; 39 | 40 | @XmlElementWrapper(name = "videos") 41 | @XmlElement(name = "video") 42 | private List masterVideos; 43 | 44 | @Data 45 | @XmlAccessorType(XmlAccessType.FIELD) 46 | public static class MasterArtistXML implements SubItemXML { 47 | 48 | @XmlElement(name = "id") 49 | private Integer artistId; 50 | 51 | @Override 52 | public MasterArtistRecord getRecord(int parentId) { 53 | return new MasterArtistRecord() 54 | .setMasterId(parentId) 55 | .setArtistId(artistId) 56 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 57 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 58 | } 59 | } 60 | 61 | @Data 62 | @XmlAccessorType(XmlAccessType.FIELD) 63 | public static class MasterVideoXML implements HashXML { 64 | 65 | @XmlElement(name = "title") 66 | private String title; 67 | 68 | @XmlElement(name = "description") 69 | private String description; 70 | 71 | @XmlAttribute(name = "src") 72 | private String url; 73 | 74 | @Override 75 | public MasterVideoRecord getRecord(int parentId) { 76 | return new MasterVideoRecord() 77 | .setTitle(title) 78 | .setDescription(description) 79 | .setUrl(url) 80 | .setHash(getHashValue()) 81 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 82 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 83 | } 84 | 85 | @Override 86 | public int getHashValue() { 87 | return makeHash(new String[]{title, description, url}); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/BatchApplicationTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.junit.jupiter.api.Assertions.assertAll; 5 | import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 6 | import static org.junit.jupiter.api.Assertions.fail; 7 | import static org.mockito.Mockito.doThrow; 8 | import static org.mockito.Mockito.times; 9 | import static org.mockito.Mockito.verify; 10 | 11 | import ch.qos.logback.classic.Level; 12 | import io.dsub.discogs.batch.testutil.LogSpy; 13 | import java.util.List; 14 | import org.junit.jupiter.api.BeforeEach; 15 | import org.junit.jupiter.api.Test; 16 | import org.junit.jupiter.api.extension.RegisterExtension; 17 | import org.mockito.ArgumentCaptor; 18 | import org.mockito.Captor; 19 | import org.mockito.Mock; 20 | import org.mockito.MockedStatic; 21 | import org.mockito.Mockito; 22 | import org.mockito.MockitoAnnotations; 23 | 24 | class BatchApplicationTest { 25 | 26 | @Mock 27 | BatchService batchService; 28 | 29 | @Captor 30 | ArgumentCaptor captor; 31 | 32 | @RegisterExtension 33 | LogSpy logSpy = new LogSpy(); 34 | 35 | @BeforeEach 36 | void setUp() { 37 | MockitoAnnotations.openMocks(this); 38 | } 39 | 40 | @Test 41 | void givenServiceThrows__WhenMainCalled__ShouldLog() { 42 | try (MockedStatic app = Mockito.mockStatic(BatchApplication.class)) { 43 | String[] args = {"hello world"}; 44 | app.when(BatchApplication::getBatchService).thenReturn(batchService); 45 | app.when(() -> BatchApplication.main(args)).thenCallRealMethod(); 46 | doThrow(new Exception(args[0])).when(batchService).run(args); 47 | BatchApplication.main(args); 48 | List errLogs = logSpy.getLogsByExactLevelAsString(Level.ERROR, true); 49 | 50 | assertAll( 51 | () -> assertThat(BatchApplication.getBatchService()).isEqualTo(batchService), 52 | () -> verify(batchService, times(1)).run(captor.capture()), 53 | () -> assertThat(captor.getValue()).isEqualTo(args), 54 | () -> assertThat(errLogs).hasSize(1).allMatch(s -> s.equals(args[0])) 55 | ); 56 | 57 | } catch (Exception e) { 58 | fail(e); 59 | } 60 | } 61 | 62 | @Test 63 | void whenGetBatchService__ShouldAlwaysReturnFreshInstance() { 64 | // when 65 | BatchService service = BatchApplication.getBatchService(); 66 | 67 | // then 68 | assertThat(service).isNotEqualTo(BatchApplication.getBatchService()); 69 | } 70 | 71 | @Test 72 | void givenAnyArg__WhenServiceExecuted__ThenReturnNormally() { 73 | try (MockedStatic app = Mockito.mockStatic(BatchApplication.class)) { 74 | String[] args = {"hello world"}; 75 | app.when(BatchApplication::getBatchService).thenReturn(batchService); 76 | app.when(() -> BatchApplication.main(args)).thenCallRealMethod(); 77 | assertDoesNotThrow(() -> BatchApplication.main(args)); 78 | app.reset(); 79 | } catch (Exception e) { 80 | fail(e); 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/config/BatchConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.config; 2 | 3 | import io.dsub.discogs.batch.job.UniqueRunIdIncrementer; 4 | import io.dsub.discogs.batch.job.listener.ClearanceJobExecutionListener; 5 | import io.dsub.discogs.batch.job.listener.ExitSignalJobExecutionListener; 6 | import io.dsub.discogs.batch.job.listener.IdCachingJobExecutionListener; 7 | import java.time.LocalDateTime; 8 | import lombok.RequiredArgsConstructor; 9 | import org.springframework.batch.core.Job; 10 | import org.springframework.batch.core.Step; 11 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 12 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 13 | import org.springframework.context.annotation.Bean; 14 | import org.springframework.context.annotation.ComponentScan; 15 | import org.springframework.context.annotation.Configuration; 16 | 17 | @Configuration 18 | @EnableBatchProcessing 19 | @ComponentScan(basePackageClasses = {DiscogsBatchConfigurer.class}) 20 | @RequiredArgsConstructor 21 | public class BatchConfig { 22 | 23 | public static final int DEFAULT_CHUNK_SIZE = 500; 24 | 25 | public static final String JOB_NAME = "discogs-batch-job" + LocalDateTime.now(); 26 | private static final String FAILED = "FAILED"; 27 | private static final String ANY = "*"; 28 | 29 | private final Step artistStep; 30 | private final Step labelStep; 31 | private final Step masterStep; 32 | private final Step releaseStep; 33 | 34 | private final JobBuilderFactory jobBuilderFactory; 35 | private final IdCachingJobExecutionListener idCachingJobExecutionListener; 36 | private final ExitSignalJobExecutionListener exitSignalJobExecutionListener; 37 | private final ClearanceJobExecutionListener clearanceJobExecutionListener; 38 | 39 | @Bean 40 | public Job discogsBatchJob() { 41 | // @formatter:off 42 | return jobBuilderFactory 43 | .get(JOB_NAME) 44 | 45 | // listeners 46 | .listener(idCachingJobExecutionListener) 47 | .listener(exitSignalJobExecutionListener) 48 | .listener(clearanceJobExecutionListener) 49 | 50 | // incrementer 51 | .incrementer(new UniqueRunIdIncrementer()) 52 | 53 | // from artist step 54 | .start(artistStep) 55 | .on(FAILED) 56 | .end() 57 | .from(artistStep) 58 | .on(ANY) 59 | .to(labelStep) 60 | 61 | // from label step 62 | .from(labelStep) 63 | .on(FAILED) 64 | .end() 65 | .from(labelStep) 66 | .on(ANY) 67 | .to(masterStep) 68 | 69 | // from master step 70 | .from(masterStep) 71 | .on(FAILED) 72 | .end() 73 | .from(masterStep) 74 | .on(ANY) 75 | .to(releaseStep) 76 | 77 | // from release item step 78 | .from(releaseStep) 79 | .on(ANY) 80 | .end() 81 | 82 | // build to conclude step flow 83 | .build() 84 | 85 | // build for job itself 86 | .build(); 87 | // @formatter:on 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/config/TaskExecutorConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.config; 2 | 3 | import static io.dsub.discogs.batch.argument.ArgType.CORE_COUNT; 4 | 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.boot.ApplicationArguments; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 10 | import oshi.SystemInfo; 11 | 12 | @Slf4j 13 | @Configuration 14 | public class TaskExecutorConfig { 15 | private static final SystemInfo SYSTEM_INFO = new SystemInfo(); 16 | 17 | /* depends on for hw information to be fetched; as physical */ 18 | private static final int PHYSICAL_CORE_SIZE = SYSTEM_INFO 19 | .getHardware() 20 | .getProcessor() 21 | .getPhysicalProcessorCount(); 22 | 23 | /* needs extra adjustments for optimizations vs convenience... */ 24 | private static final int DEFAULT_THROTTLE_LIMIT = PHYSICAL_CORE_SIZE > 2 ? (int) ( 25 | PHYSICAL_CORE_SIZE * 0.8) : 1; 26 | 27 | /** 28 | * Primary bean for batch processing. The core and max pool size is fit into same value, just as 29 | * the same as the core size of host processor count. 30 | * 31 | *

{@link ThreadPoolTaskExecutor#setWaitForTasksToCompleteOnShutdown(boolean)}} is set to 32 | * true, 33 | * in case of additional tasks required after the actual job is doe. 34 | * 35 | * @return instance of {@link ThreadPoolTaskExecutor}. 36 | */ 37 | @Bean 38 | public ThreadPoolTaskExecutor batchTaskExecutor(ApplicationArguments args) { 39 | int coreCount = getCoreCount(args); 40 | ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); 41 | taskExecutor.setCorePoolSize(coreCount); 42 | taskExecutor.setMaxPoolSize(coreCount); 43 | taskExecutor.setWaitForTasksToCompleteOnShutdown(true); 44 | taskExecutor.afterPropertiesSet(); 45 | taskExecutor.initialize(); 46 | return taskExecutor; 47 | } 48 | 49 | private int getCoreCount(ApplicationArguments args) { 50 | int coreCount = DEFAULT_THROTTLE_LIMIT; 51 | if (args.containsOption(CORE_COUNT.getGlobalName())) { 52 | int givenCnt = Integer.parseInt(args.getOptionValues(CORE_COUNT.getGlobalName()).get(0)); 53 | log.info("found core count argument: {}", givenCnt); 54 | coreCount = getValidatedCoreCount(givenCnt); 55 | } 56 | log.info("setting core count to {}.", coreCount); 57 | return coreCount; 58 | } 59 | 60 | private int getValidatedCoreCount(int givenCount) { 61 | int coreCount = DEFAULT_THROTTLE_LIMIT; 62 | if (givenCount > PHYSICAL_CORE_SIZE) { 63 | log.info( 64 | "given core count {} exceeds logical core size. reducing core usage to {} from throttle..", 65 | givenCount, 66 | DEFAULT_THROTTLE_LIMIT); 67 | } else if (givenCount < 0) { 68 | log.info( 69 | "core count found negative value: {}. using default core setting: {}.", 70 | givenCount, 71 | DEFAULT_THROTTLE_LIMIT); 72 | } else { 73 | coreCount = givenCount; 74 | } 75 | return coreCount; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/step/AbstractStepConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.step; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.mockito.ArgumentMatchers.any; 5 | import static org.mockito.Mockito.doReturn; 6 | import static org.mockito.Mockito.when; 7 | 8 | import ch.qos.logback.classic.Level; 9 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 10 | import io.dsub.discogs.batch.testutil.LogSpy; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | import org.junit.jupiter.api.BeforeEach; 14 | import org.junit.jupiter.api.extension.RegisterExtension; 15 | import org.junit.jupiter.params.ParameterizedTest; 16 | import org.junit.jupiter.params.provider.ValueSource; 17 | import org.mockito.Mockito; 18 | import org.springframework.batch.core.ExitStatus; 19 | import org.springframework.batch.core.JobExecution; 20 | import org.springframework.batch.core.JobParameter; 21 | import org.springframework.batch.core.JobParameters; 22 | import org.springframework.batch.core.job.flow.FlowExecutionStatus; 23 | import org.springframework.batch.core.job.flow.JobExecutionDecider; 24 | 25 | class AbstractStepConfigTest { 26 | 27 | AbstractStepConfig stepConfig; 28 | 29 | @RegisterExtension 30 | LogSpy logSpy = new LogSpy(); 31 | 32 | @BeforeEach 33 | void setUp() throws InvalidArgumentException { 34 | stepConfig = Mockito.mock(AbstractStepConfig.class); 35 | when(stepConfig.executionDecider(any())).thenCallRealMethod(); 36 | } 37 | 38 | @ParameterizedTest 39 | @ValueSource(strings = {"artist", "release", "master", "label"}) 40 | void whenGetOnKeyExecutionDecider__ShouldReturnValidExecutionDecider(String param) 41 | throws InvalidArgumentException { 42 | 43 | JobExecutionDecider jobExecutionDecider = stepConfig.executionDecider(param); 44 | 45 | JobExecution jobExecution = Mockito.mock(JobExecution.class); 46 | JobParameters jobParameters = Mockito.mock(JobParameters.class); 47 | 48 | Map falsyMap = new HashMap<>(); 49 | Map truthyMap = new HashMap<>(); 50 | truthyMap.put(param, new JobParameter("hello")); 51 | 52 | ExitStatus exitStatus = Mockito.mock(ExitStatus.class); 53 | doReturn("COMPLETED").when(exitStatus).getExitCode(); 54 | doReturn(exitStatus).when(jobExecution).getExitStatus(); 55 | 56 | when(jobExecution.getJobParameters()).thenReturn(jobParameters); 57 | when(jobParameters.getParameters()).thenReturn(falsyMap); 58 | 59 | FlowExecutionStatus status = jobExecutionDecider.decide(jobExecution, null); 60 | assertThat(status.getName()).isEqualTo("SKIPPED"); 61 | 62 | if (logSpy.countExact(Level.DEBUG) > 0) { 63 | assertThat(logSpy.getLogsByLevelAsString(Level.DEBUG, true).get(0)) 64 | .contains(param, "skipping"); 65 | logSpy.clear(); 66 | } 67 | 68 | when(jobParameters.getParameters()).thenReturn(truthyMap); 69 | status = jobExecutionDecider.decide(jobExecution, null); 70 | 71 | assertThat(status.getName()).isEqualTo("COMPLETED"); 72 | if (logSpy.countExact(Level.DEBUG) > 0) { 73 | assertThat(logSpy.getLogsByLevelAsString(Level.DEBUG, true).get(0)) 74 | .contains(param, "executing"); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/job/processor/ItemProcessorConfig.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.processor; 2 | 3 | import io.dsub.discogs.batch.domain.artist.ArtistSubItemsXML; 4 | import io.dsub.discogs.batch.domain.artist.ArtistXML; 5 | import io.dsub.discogs.batch.domain.label.LabelSubItemsXML; 6 | import io.dsub.discogs.batch.domain.label.LabelXML; 7 | import io.dsub.discogs.batch.domain.master.MasterMainReleaseXML; 8 | import io.dsub.discogs.batch.domain.master.MasterSubItemsXML; 9 | import io.dsub.discogs.batch.domain.master.MasterXML; 10 | import io.dsub.discogs.batch.domain.release.ReleaseItemSubItemsXML; 11 | import io.dsub.discogs.batch.domain.release.ReleaseItemXML; 12 | import io.dsub.discogs.batch.job.registry.DefaultEntityIdRegistry; 13 | import io.dsub.discogs.jooq.tables.records.ArtistRecord; 14 | import io.dsub.discogs.jooq.tables.records.LabelRecord; 15 | import io.dsub.discogs.jooq.tables.records.MasterRecord; 16 | import io.dsub.discogs.jooq.tables.records.ReleaseItemRecord; 17 | import java.util.Collection; 18 | import lombok.RequiredArgsConstructor; 19 | import org.jooq.UpdatableRecord; 20 | import org.springframework.batch.core.configuration.annotation.StepScope; 21 | import org.springframework.batch.item.ItemProcessor; 22 | import org.springframework.context.annotation.Bean; 23 | import org.springframework.context.annotation.Configuration; 24 | 25 | @Configuration 26 | @RequiredArgsConstructor 27 | public class ItemProcessorConfig { 28 | 29 | private final DefaultEntityIdRegistry entityIdRegistry; 30 | 31 | @Bean 32 | @StepScope 33 | public ItemProcessor artistCoreProcessor() { 34 | return new ArtistCoreProcessor(); 35 | } 36 | 37 | @Bean 38 | @StepScope 39 | public ItemProcessor>> 40 | artistSubItemsProcessor() { 41 | return new ArtistSubItemsProcessor(entityIdRegistry); 42 | } 43 | 44 | @Bean 45 | @StepScope 46 | public ItemProcessor labelCoreProcessor() { 47 | return new LabelCoreProcessor(); 48 | } 49 | 50 | @Bean 51 | @StepScope 52 | public ItemProcessor>> labelSubItemsProcessor() { 53 | return new LabelSubItemsProcessor(entityIdRegistry); 54 | } 55 | 56 | @Bean 57 | @StepScope 58 | public ItemProcessor masterCoreProcessor() { 59 | return new MasterCoreProcessor(); 60 | } 61 | 62 | @Bean 63 | @StepScope 64 | public ItemProcessor>> 65 | masterSubItemsProcessor() { 66 | return new MasterSubItemsProcessor(entityIdRegistry); 67 | } 68 | 69 | @Bean 70 | @StepScope 71 | public ItemProcessor releaseItemCoreProcessor() { 72 | return new ReleaseItemCoreProcessor(entityIdRegistry); 73 | } 74 | 75 | @Bean 76 | @StepScope 77 | public ItemProcessor>> 78 | releaseItemSubItemsProcessor() { 79 | return new ReleaseItemSubItemsProcessor(entityIdRegistry); 80 | } 81 | 82 | @Bean 83 | @StepScope 84 | public ItemProcessor masterMainReleaseItemProcessor() { 85 | return new MasterMainReleaseItemProcessor(entityIdRegistry); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/validator/CompositeArgumentValidator.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import io.dsub.discogs.batch.exception.MissingRequiredParamsException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | import org.springframework.beans.factory.InitializingBean; 7 | import org.springframework.boot.ApplicationArguments; 8 | 9 | /** 10 | * Composite argument validator that delegates its validation to list of other validators. Can be 11 | * used to tread a group of validators as a single validator. Also, a group of validators can also 12 | * be accumulated accordingly. 13 | */ 14 | public class CompositeArgumentValidator implements ArgumentValidator, InitializingBean { 15 | 16 | /** 17 | * A list of delegates that will actually perform. 18 | */ 19 | private final List delegates = new ArrayList<>(); 20 | 21 | /** 22 | * Adds validator to its delegates list. 23 | * 24 | * @param delegate a validator to delegate. 25 | * @return itself. 26 | */ 27 | public CompositeArgumentValidator addValidator(ArgumentValidator delegate) { 28 | if (delegate == null) { 29 | return this; 30 | } 31 | this.delegates.add(delegate); 32 | return this; 33 | } 34 | 35 | /** 36 | * Adds list of validators to its delegates list. 37 | * 38 | * @param delegates additional list of delegates to be added. 39 | * @return itself. 40 | */ 41 | public CompositeArgumentValidator addValidators(List delegates) { 42 | if (delegates == null) { 43 | return this; 44 | } 45 | this.delegates.addAll(delegates); 46 | return this; 47 | } 48 | 49 | /** 50 | * Adds unbounded numbers of validators to its delegates list. 51 | * 52 | * @param delegates additional validators to be added (or can be null) 53 | * @return itself. 54 | */ 55 | public CompositeArgumentValidator addValidators(ArgumentValidator... delegates) { 56 | if (delegates == null || delegates.length == 0) { 57 | return this; 58 | } 59 | this.delegates.addAll(List.of(delegates)); 60 | return this; 61 | } 62 | 63 | /** 64 | * Method to delegate validations. 65 | * 66 | * @param applicationArguments to be validated. 67 | * @return result of validation set accumulated by validators. 68 | */ 69 | @Override 70 | public ValidationResult validate(ApplicationArguments applicationArguments) { 71 | if (applicationArguments == null) { 72 | return new DefaultValidationResult("applicationArgument cannot be null"); 73 | } 74 | ValidationResult result = new DefaultValidationResult(); 75 | for (ArgumentValidator delegate : delegates) { 76 | result = result.combine(delegate.validate(applicationArguments)); 77 | if (!result.isValid()) { 78 | return result; 79 | } 80 | } 81 | return result; 82 | } 83 | 84 | /** 85 | * Method to fulfill {@link InitializingBean}. 86 | * 87 | * @throws MissingRequiredParamsException thrown if delegates are empty (cannot be null.) 88 | */ 89 | @Override 90 | public void afterPropertiesSet() throws MissingRequiredParamsException { 91 | if (this.delegates.isEmpty()) { 92 | throw new MissingRequiredParamsException("delegates must not be null or empty"); 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/validator/CompositeArgumentValidatorUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 5 | import static org.junit.jupiter.api.Assertions.assertThrows; 6 | 7 | import io.dsub.discogs.batch.exception.MissingRequiredParamsException; 8 | import java.util.List; 9 | import org.junit.jupiter.api.BeforeEach; 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.boot.DefaultApplicationArguments; 12 | 13 | class CompositeArgumentValidatorUnitTest { 14 | 15 | private CompositeArgumentValidator compositeArgumentValidator; 16 | private ArgumentValidator helloValidator; 17 | private ArgumentValidator worldValidator; 18 | 19 | @BeforeEach 20 | void setUp() { 21 | this.compositeArgumentValidator = new CompositeArgumentValidator(); 22 | this.helloValidator = arg -> new DefaultValidationResult("hello"); 23 | this.worldValidator = arg -> new DefaultValidationResult("world"); 24 | } 25 | 26 | @Test 27 | void addValidator() { 28 | assertThat(this.compositeArgumentValidator.addValidator(helloValidator)) 29 | .isEqualTo(this.compositeArgumentValidator); 30 | assertThat(this.compositeArgumentValidator.addValidator(worldValidator)) 31 | .isEqualTo(this.compositeArgumentValidator); 32 | } 33 | 34 | @Test 35 | void addValidators() { 36 | this.compositeArgumentValidator = new CompositeArgumentValidator(); 37 | assertThat(this.compositeArgumentValidator.addValidators(helloValidator, worldValidator)) 38 | .isEqualTo(this.compositeArgumentValidator); 39 | assertThat( 40 | this.compositeArgumentValidator.addValidators(List.of(helloValidator, worldValidator))) 41 | .isEqualTo(this.compositeArgumentValidator); 42 | List argumentValidators = null; 43 | assertThat(this.compositeArgumentValidator.addValidators(argumentValidators)) 44 | .isEqualTo(this.compositeArgumentValidator); 45 | ArgumentValidator[] validators = null; 46 | assertThat(this.compositeArgumentValidator.addValidators(validators)) 47 | .isEqualTo(this.compositeArgumentValidator); 48 | } 49 | 50 | @Test 51 | void validate() { 52 | this.compositeArgumentValidator.addValidators(helloValidator, worldValidator); 53 | ValidationResult result = this.compositeArgumentValidator.validate(null); 54 | assertThat(result.isValid()).isFalse(); 55 | assertThat(result.getIssues().size()).isEqualTo(1); 56 | assertThat(result.getIssues().get(0)).isEqualTo("applicationArgument cannot be null"); 57 | 58 | result = this.compositeArgumentValidator.validate(new DefaultApplicationArguments()); 59 | assertThat(result.isValid()).isFalse(); 60 | assertThat(result.getIssues().size()).isEqualTo(1); 61 | assertThat(result.getIssues().get(0)).isEqualTo("hello"); 62 | } 63 | 64 | @Test 65 | void afterPropertiesSet() { 66 | assertThrows( 67 | MissingRequiredParamsException.class, 68 | () -> this.compositeArgumentValidator.afterPropertiesSet()); 69 | this.compositeArgumentValidator.addValidator(helloValidator); 70 | assertDoesNotThrow(() -> this.compositeArgumentValidator.afterPropertiesSet()); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/tasklet/FileClearTaskletTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.tasklet; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.junit.jupiter.api.Assertions.fail; 5 | import static org.mockito.BDDMockito.given; 6 | import static org.mockito.BDDMockito.willThrow; 7 | import static org.mockito.Mockito.never; 8 | import static org.mockito.Mockito.times; 9 | import static org.mockito.Mockito.verify; 10 | 11 | import io.dsub.discogs.batch.exception.FileException; 12 | import io.dsub.discogs.batch.util.FileUtil; 13 | import org.junit.jupiter.api.BeforeEach; 14 | import org.junit.jupiter.api.Test; 15 | import org.mockito.InjectMocks; 16 | import org.mockito.Mock; 17 | import org.mockito.MockitoAnnotations; 18 | import org.springframework.batch.core.JobExecution; 19 | import org.springframework.batch.core.StepContribution; 20 | import org.springframework.batch.core.StepExecution; 21 | import org.springframework.batch.core.scope.context.ChunkContext; 22 | import org.springframework.batch.core.scope.context.StepContext; 23 | 24 | class FileClearTaskletTest { 25 | 26 | final StepExecution stepExecution = new StepExecution("step", new JobExecution(1L)); 27 | final ChunkContext chunkContext = new ChunkContext(new StepContext(stepExecution)); 28 | final StepContribution stepContribution = new StepContribution(stepExecution); 29 | 30 | @InjectMocks 31 | FileClearTasklet fileClearTasklet; 32 | 33 | @Mock 34 | FileUtil fileUtil; 35 | 36 | @BeforeEach 37 | void setUp() { 38 | MockitoAnnotations.openMocks(this); 39 | } 40 | 41 | @Test 42 | void givenFilesAreTemporary__WhenTaskExecutes__ShouldCallClearAll() { 43 | try { 44 | // given 45 | given(fileUtil.isTemporary()).willReturn(true); 46 | 47 | // when 48 | fileClearTasklet.execute(stepContribution, chunkContext); 49 | 50 | // then 51 | verify(fileUtil, times(1)).clearAll(); 52 | } catch (FileException e) { 53 | fail(e); 54 | } 55 | } 56 | 57 | @Test 58 | void givenFilesAreMounted__WhenTaskExecutes__ShouldNotCallClearAll() { 59 | try { 60 | // given 61 | given(fileUtil.isTemporary()).willReturn(false); 62 | 63 | // when 64 | fileClearTasklet.execute(stepContribution, chunkContext); 65 | 66 | // then 67 | verify(fileUtil, never()).clearAll(); 68 | } catch (FileException e) { 69 | fail(e); 70 | } 71 | } 72 | 73 | @Test 74 | void givenFilesAreMounted__WhenTaskExecutes__ShouldMarkedAsComplete() { 75 | // given 76 | given(fileUtil.isTemporary()).willReturn(false); 77 | 78 | // when 79 | fileClearTasklet.execute(stepContribution, chunkContext); 80 | 81 | // then 82 | assertThat(chunkContext.isComplete()).isTrue(); 83 | assertThat(stepContribution.getExitStatus().getExitCode()).isEqualTo("COMPLETED"); 84 | } 85 | 86 | @Test 87 | void givenClearAllThrows__WhenTaskExecutes__WillMarkAsComplete() { 88 | try { 89 | // given 90 | willThrow(new FileException("FAIL")).given(fileUtil).clearAll(); 91 | 92 | // when 93 | fileClearTasklet.execute(stepContribution, chunkContext); 94 | 95 | // then 96 | assertThat(chunkContext.isComplete()).isTrue(); 97 | assertThat(stepContribution.getExitStatus().getExitCode()).isEqualTo("COMPLETED"); 98 | } catch (FileException e) { 99 | fail(e); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/job/listener/ItemCountingItemProcessListenerTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.job.listener; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.mockito.BDDMockito.given; 5 | import static org.mockito.Mockito.never; 6 | import static org.mockito.Mockito.times; 7 | import static org.mockito.Mockito.verify; 8 | 9 | import ch.qos.logback.classic.Level; 10 | import io.dsub.discogs.batch.testutil.LogSpy; 11 | import java.util.ArrayList; 12 | import java.util.Collection; 13 | import java.util.HashSet; 14 | import java.util.List; 15 | import java.util.Set; 16 | import java.util.concurrent.atomic.AtomicLong; 17 | import org.junit.jupiter.api.BeforeEach; 18 | import org.junit.jupiter.api.Test; 19 | import org.junit.jupiter.api.extension.RegisterExtension; 20 | import org.mockito.Mock; 21 | import org.mockito.Mockito; 22 | 23 | class ItemCountingItemProcessListenerTest { 24 | 25 | private ItemCountingItemProcessListener listener; 26 | private AtomicLong counter; 27 | 28 | @RegisterExtension 29 | LogSpy logSpy = new LogSpy(); 30 | 31 | @BeforeEach 32 | void setUp() { 33 | this.counter = Mockito.spy(new AtomicLong(0)); 34 | this.listener = Mockito.spy(new ItemCountingItemProcessListener(counter)); 35 | } 36 | 37 | @Test 38 | void givenItemIsList__ShouldCountProperly() { 39 | List items = List.of(1, 2, 3, 4, 5); 40 | listener.afterProcess(items, items); 41 | assertThat(counter.get()).isEqualTo(5); 42 | } 43 | 44 | @Test 45 | void whenCount__ShouldOnlyTouchResult() { 46 | List item = Mockito.mock(List.class); 47 | List result = Mockito.mock(List.class); 48 | given(result.size()).willReturn(5); 49 | 50 | listener.afterProcess(item, result); 51 | 52 | verify(item, times(0)).size(); 53 | verify(result, times(1)).size(); 54 | assertThat(counter.get()).isEqualTo(result.size()); 55 | } 56 | 57 | @Test 58 | void whenCount__ShouldCountIfResultIsCollection() { 59 | Collection item = Mockito.mock(Collection.class); 60 | given(item.size()).willReturn(101); 61 | 62 | listener.afterProcess(item, item); 63 | 64 | assertThat(counter.get()).isEqualTo(item.size()); 65 | } 66 | 67 | @Test 68 | void whenCount__ShouldCountIfNotCollection() { 69 | Object object = Mockito.mock(Object.class); 70 | listener.afterProcess(object, object); 71 | assertThat(counter.get()).isEqualTo(1); 72 | } 73 | 74 | @Test 75 | void whenCount__ShouldSkipNullResult() { 76 | listener.afterProcess(null, null); 77 | assertThat(counter.get()).isZero(); 78 | } 79 | 80 | @Test 81 | void whenCount__ShouldCountSetResult() { 82 | Set items = Set.of(1,2,3,4,5); 83 | listener.afterProcess(items, items); 84 | assertThat(counter.get()).isEqualTo(5); 85 | } 86 | 87 | @Test 88 | void whenBeforeProcess__ShouldNoOp() { 89 | List list = Mockito.spy(List.of()); 90 | listener.beforeProcess(list); 91 | verify(list, never()).size(); 92 | } 93 | 94 | @Test 95 | void whenErrorOccur__ShouldLog() { 96 | Exception e = new Exception("test error"); 97 | String item = "test item"; 98 | listener.onProcessError(item, e); 99 | List logs = logSpy.getLogsByLevelAsString(Level.ERROR, true); 100 | assertThat(logs).hasSize(1); 101 | String log = logs.get(0); 102 | assertThat(log).contains("test item", "test error"); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/domain/artist/ArtistSubItemsXML.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.domain.artist; 2 | 3 | import io.dsub.discogs.batch.domain.SubItemXML; 4 | import io.dsub.discogs.jooq.tables.records.ArtistAliasRecord; 5 | import io.dsub.discogs.jooq.tables.records.ArtistGroupRecord; 6 | import io.dsub.discogs.jooq.tables.records.ArtistMemberRecord; 7 | import java.time.Clock; 8 | import java.time.LocalDateTime; 9 | import java.util.List; 10 | import javax.xml.bind.annotation.XmlAccessType; 11 | import javax.xml.bind.annotation.XmlAccessorType; 12 | import javax.xml.bind.annotation.XmlAttribute; 13 | import javax.xml.bind.annotation.XmlElement; 14 | import javax.xml.bind.annotation.XmlElementWrapper; 15 | import javax.xml.bind.annotation.XmlRootElement; 16 | import lombok.Data; 17 | import lombok.EqualsAndHashCode; 18 | 19 | @Data 20 | @XmlRootElement(name = "artist") 21 | @XmlAccessorType(XmlAccessType.FIELD) 22 | @EqualsAndHashCode(callSuper = false) 23 | public class ArtistSubItemsXML { 24 | 25 | boolean prepared = false; 26 | 27 | @XmlElement(name = "id") 28 | private Integer id; 29 | 30 | @XmlElementWrapper(name = "aliases") 31 | @XmlElement(name = "name") 32 | private List aliases; 33 | 34 | @XmlElementWrapper(name = "groups") 35 | @XmlElement(name = "name") 36 | private List groups; 37 | 38 | @XmlElementWrapper(name = "members") 39 | @XmlElement(name = "name") 40 | private List members; 41 | 42 | @XmlElementWrapper(name = "namevariations") 43 | @XmlElement(name = "name") 44 | private List nameVariations; 45 | 46 | @XmlElementWrapper(name = "urls") 47 | @XmlElement(name = "url") 48 | private List urls; 49 | 50 | @Data 51 | @XmlAccessorType(XmlAccessType.FIELD) 52 | public static class ArtistAliasXML implements SubItemXML { 53 | 54 | @XmlAttribute(name = "id") 55 | private Integer aliasId; 56 | 57 | @Override 58 | public ArtistAliasRecord getRecord(int parentId) { 59 | return new ArtistAliasRecord() 60 | .setArtistId(parentId) 61 | .setAliasId(aliasId) 62 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 63 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 64 | } 65 | } 66 | 67 | @Data 68 | @XmlAccessorType(XmlAccessType.FIELD) 69 | public static class ArtistGroupXML implements SubItemXML { 70 | 71 | @XmlAttribute(name = "id") 72 | private Integer groupId; 73 | 74 | @Override 75 | public ArtistGroupRecord getRecord(int parentId) { 76 | return new ArtistGroupRecord() 77 | .setArtistId(parentId) 78 | .setGroupId(groupId) 79 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 80 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 81 | } 82 | } 83 | 84 | @Data 85 | @XmlAccessorType(XmlAccessType.FIELD) 86 | public static class ArtistMemberXML implements SubItemXML { 87 | 88 | @XmlAttribute(name = "id") 89 | private Integer memberId; 90 | 91 | @Override 92 | public ArtistMemberRecord getRecord(int parentId) { 93 | return new ArtistMemberRecord() 94 | .setArtistId(parentId) 95 | .setMemberId(memberId) 96 | .setCreatedAt(LocalDateTime.now(Clock.systemUTC())) 97 | .setLastModifiedAt(LocalDateTime.now(Clock.systemUTC())); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/testutil/LogSpy.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.testutil; 2 | 3 | import ch.qos.logback.classic.Level; 4 | import ch.qos.logback.classic.spi.ILoggingEvent; 5 | import ch.qos.logback.core.read.ListAppender; 6 | import java.util.List; 7 | import java.util.stream.Collectors; 8 | import org.junit.jupiter.api.extension.AfterEachCallback; 9 | import org.junit.jupiter.api.extension.BeforeEachCallback; 10 | import org.junit.jupiter.api.extension.ExtensionContext; 11 | import org.junit.jupiter.api.extension.TestInstantiationException; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | public class LogSpy implements BeforeEachCallback, AfterEachCallback { 16 | 17 | private ch.qos.logback.classic.Logger logger; 18 | private ListAppender appender; 19 | 20 | @Override 21 | public void afterEach(ExtensionContext context) { 22 | logger.detachAppender(appender); 23 | } 24 | 25 | @Override 26 | public void beforeEach(ExtensionContext context) { 27 | appender = new ListAppender<>(); 28 | logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); 29 | logger.addAppender(appender); 30 | appender.start(); 31 | } 32 | 33 | public List getEvents() throws TestInstantiationException { 34 | if (appender == null) { 35 | throw new TestInstantiationException("LogSpy needs to be annotated with @Rule"); 36 | } 37 | return appender.list; 38 | } 39 | 40 | public List getLogsByLevel(Level lv) throws TestInstantiationException { 41 | return getEvents().stream() 42 | .filter(log -> log.getLevel().isGreaterOrEqual(lv)) 43 | .collect(Collectors.toList()); 44 | } 45 | 46 | public List getLogsByLevelExact(Level lv) { 47 | return getEvents().stream() 48 | .filter(log -> log.getLevel().equals(lv)) 49 | .collect(Collectors.toList()); 50 | } 51 | 52 | public List getLogsByLevelExact(Level lv, String targetPackage) { 53 | return getEvents().stream() 54 | .filter(log -> log.getLevel().equals(lv)) 55 | .filter(log -> log.getLoggerName().contains(targetPackage)) 56 | .collect(Collectors.toList()); 57 | } 58 | 59 | public List getLogsAsString(boolean formatted) { 60 | return getEvents().stream() 61 | .map(log -> formatted ? log.getFormattedMessage() : log.getMessage()) 62 | .collect(Collectors.toList()); 63 | } 64 | 65 | public List getLogsByLevelAsString(Level lv, boolean formatted) { 66 | return getLogsByLevel(lv).stream() 67 | .map(log -> formatted ? log.getFormattedMessage() : log.getMessage()) 68 | .collect(Collectors.toList()); 69 | } 70 | 71 | public List getLogsByExactLevelAsString(Level lv, boolean formatted) { 72 | return getLogsByLevelExact(lv).stream() 73 | .map(log -> formatted ? log.getFormattedMessage() : log.getMessage()) 74 | .collect(Collectors.toList()); 75 | } 76 | 77 | public List getLogsByExactLevelAsString(Level lv, boolean formatted, String basePackage) { 78 | return getLogsByLevelExact(lv, basePackage).stream() 79 | .map(log -> formatted ? log.getFormattedMessage() : log.getMessage()) 80 | .collect(Collectors.toList()); 81 | } 82 | 83 | public int count() { 84 | return getEvents().size(); 85 | } 86 | 87 | public int count(Level level) { 88 | return getLogsByLevel(level).size(); 89 | } 90 | 91 | public int countExact(Level level) { 92 | return getLogsByLevelExact(level).size(); 93 | } 94 | 95 | public void clear() { 96 | appender.list.clear(); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/formatter/JdbcUrlFormatter.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.formatter; 2 | 3 | import io.dsub.discogs.batch.datasource.DBType; 4 | import java.util.Arrays; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | import lombok.extern.slf4j.Slf4j; 8 | 9 | /** 10 | * A convenience class that performs jdbc url formatter. Will remain deprecated until need arises 11 | * again, or tbd to be removed. 12 | */ 13 | @Slf4j 14 | public class JdbcUrlFormatter implements ArgumentFormatter { 15 | 16 | private static final Pattern JDBC_URL_PATTERN = Pattern.compile( 17 | "jdbc:(\\w+)://[.\\w]+:[\\d]+(/\\w+)?.*" 18 | ); 19 | 20 | private static final Pattern JDBC_OPTION_PATTERN = Pattern.compile(".*(\\?).*"); 21 | 22 | /* JDBC CONNECTION OPTIONS */ 23 | private static final String TIME_ZONE_UTC_OPT = "serverTimeZone=UTC"; 24 | private static final String CACHE_PREP_STMT_OPT = "cachePrepStmts=true"; 25 | private static final String USE_SERVER_PREP_STMTS_OPT = "useServerPrepStmts=true"; 26 | private static final String REWRITE_BATCHED_STMTS_OPT = "rewriteBatchedStatements=true"; 27 | private static final String NO_LEGACY_DATE_TIME_CODE_OPT = "useLegacyDatetimeCode=false"; 28 | 29 | /* HEADER */ 30 | private static final String URL_HEADER = "url="; 31 | 32 | @Override 33 | public String[] format(String[] args) { 34 | if (args == null || args.length == 0) { 35 | return args; 36 | } 37 | return Arrays.stream(args) 38 | .map(arg -> arg.startsWith(URL_HEADER) ? doFormat(arg) : arg) 39 | .toArray(String[]::new); 40 | } 41 | 42 | private String doFormat(String url) { 43 | if (url == null || url.isBlank()) { 44 | return null; 45 | } 46 | 47 | url = url.replace(URL_HEADER, ""); // removes header 48 | 49 | Matcher m = JDBC_URL_PATTERN.matcher(url); 50 | 51 | boolean patternMatches = m.matches(); 52 | 53 | if (patternMatches && (m.group(2) == null || m.group(2).isBlank())) { 54 | log.info( 55 | "default database or schema missing. appending default schema \"discogs\" to jdbc url"); 56 | 57 | String[] parts = url.split("\\?"); 58 | 59 | url = parts[0] + "/discogs"; 60 | 61 | if (parts.length > 1) { 62 | url = url + "?" + parts[1]; 63 | } 64 | } 65 | 66 | if (patternMatches) { 67 | String databaseProductName = m.group(1); 68 | DBType type = DBType.getTypeOf(databaseProductName); 69 | if (type == null) { 70 | return URL_HEADER + url; 71 | } 72 | } 73 | 74 | return URL_HEADER + appendOptions(url); 75 | } 76 | 77 | private boolean isOptionPresent(String url) { 78 | return JDBC_OPTION_PATTERN.matcher(url).matches(); 79 | } 80 | 81 | private String appendOptions(String url) { 82 | String amp = "&"; 83 | String q = "?"; 84 | String header = isOptionPresent(url) ? amp : q; 85 | 86 | if (!url.matches(".*(?i)serverTimezone.*")) { 87 | url += header + TIME_ZONE_UTC_OPT; 88 | header = amp; 89 | } 90 | if (!url.matches(".*(?i)(cachePrepStmts).*")) { 91 | url += header + CACHE_PREP_STMT_OPT; 92 | header = amp; 93 | } 94 | if (!url.matches(".*(?i)rewriteBatchedStatements.*")) { 95 | url += header + REWRITE_BATCHED_STMTS_OPT; 96 | header = amp; 97 | } 98 | if (!url.matches(".*(?i)(useServerPrepStmts).*")) { 99 | url += header + USE_SERVER_PREP_STMTS_OPT; 100 | header = amp; 101 | } 102 | if (!url.matches(".*(?i)(useLegacyDatetimeCode).*")) { 103 | url += header + NO_LEGACY_DATE_TIME_CODE_OPT; 104 | } 105 | 106 | return url; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/argument/ArgType.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument; 2 | 3 | import java.util.List; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | /** 7 | * Enum to represent current supported argument types. 8 | */ 9 | @RequiredArgsConstructor 10 | public enum ArgType { 11 | CHUNK_SIZE( 12 | ArgumentProperty.builder() 13 | .globalName("chunkSize") 14 | .supportedType(Long.class) 15 | .synonyms("chunk", "c") 16 | .build()), 17 | CORE_COUNT( 18 | ArgumentProperty.builder() 19 | .globalName("coreCount") 20 | .synonyms("core") 21 | .supportedType(Long.class) 22 | .build()), 23 | ETAG(ArgumentProperty.builder().globalName("eTag").synonyms("e").maxValuesCount(4).build()), 24 | MOUNT( 25 | ArgumentProperty.builder() 26 | .globalName("mount") 27 | .synonyms("m") 28 | .required(false) 29 | .maxValuesCount(0) 30 | .minValuesCount(0) 31 | .build()), 32 | PASSWORD( 33 | ArgumentProperty.builder() 34 | .globalName("password") 35 | .synonyms("password", "pass", "p") 36 | .required(true) 37 | .build()), 38 | STRICT( 39 | ArgumentProperty.builder() 40 | .globalName("strict") 41 | .synonyms("s") 42 | .required(false) 43 | .maxValuesCount(0) 44 | .minValuesCount(0) 45 | .build()), 46 | TYPE(ArgumentProperty.builder().globalName("type").synonyms("t").maxValuesCount(4).build()), 47 | URL(ArgumentProperty.builder().globalName("url").required(true).build()), 48 | USERNAME( 49 | ArgumentProperty.builder() 50 | .globalName("username") 51 | .synonyms("username", "user", "u") 52 | .required(true) 53 | .build()), 54 | YEAR( 55 | ArgumentProperty.builder() 56 | .globalName("year") 57 | .synonyms("y") 58 | .supportedType(Long.class) 59 | .build()), 60 | DRIVER_CLASS_NAME( 61 | ArgumentProperty.builder() 62 | .globalName("driverClassName") 63 | .synonyms("driverclassname", "driver_class_name") 64 | .required(false) 65 | .supportedType(String.class) 66 | .build()), 67 | YEAR_MONTH(ArgumentProperty.builder().globalName("yearMonth").synonyms("ym").build()); 68 | 69 | // properties mapped to each enum instance. 70 | private final ArgumentProperty props; 71 | 72 | public static ArgType getTypeOf(String key) { 73 | if (key == null || key.isBlank()) { 74 | return null; 75 | } 76 | String target = key.toLowerCase(); 77 | for (ArgType argType : ArgType.values()) { 78 | if (argType.props.contains(target)) { 79 | return argType; 80 | } 81 | } 82 | return null; 83 | } 84 | 85 | public static boolean contains(String key) { 86 | for (ArgType t : ArgType.values()) { 87 | if (t.props.contains(key)) { 88 | return true; 89 | } 90 | } 91 | return false; 92 | } 93 | 94 | public List getSynonyms() { 95 | return List.copyOf(this.props.getSynonyms()); 96 | } 97 | 98 | public boolean isValueRequired() { 99 | return this.props.getMinValuesCount() > 0; 100 | } 101 | 102 | public int getMinValuesCount() { 103 | return this.props.getMinValuesCount(); 104 | } 105 | 106 | public int getMaxValuesCount() { 107 | return this.props.getMaxValuesCount(); 108 | } 109 | 110 | public String getGlobalName() { 111 | return this.props.getGlobalName(); 112 | } 113 | 114 | public boolean isRequired() { 115 | return this.props.isRequired(); 116 | } 117 | 118 | public Class getSupportedType() { 119 | return this.props.getSupportedType(); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/validator/DataSourceArgumentValidatorUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import java.util.List; 6 | import org.junit.jupiter.api.Test; 7 | import org.springframework.boot.DefaultApplicationArguments; 8 | 9 | class DataSourceArgumentValidatorUnitTest { 10 | 11 | private final DataSourceArgumentValidator validator = new DataSourceArgumentValidator(); 12 | 13 | void innerTestCorrectJdbcUrl(String jdbcUrl) { 14 | String[] args = new String[]{jdbcUrl, "user=hello", "pass=pass"}; 15 | ValidationResult result = validator.validate(new DefaultApplicationArguments(args)); 16 | assertThat(result.isValid()).isEqualTo(true); 17 | } 18 | 19 | void innerTestMalformedJdbcUrl(String jdbcUrl, String expectedReport) { 20 | String[] args = new String[]{jdbcUrl, "user=hello", "pass=pass"}; 21 | ValidationResult result = validator.validate(new DefaultApplicationArguments(args)); 22 | assertThat(result.isValid()).isEqualTo(false); 23 | assertThat(expectedReport).isIn(result.getIssues()); 24 | } 25 | 26 | @Test 27 | void shouldReportEveryMissingFields() { 28 | String[] arg = new String[0]; 29 | ValidationResult result = validator.validate(new DefaultApplicationArguments(arg)); 30 | 31 | assertThat(result).returns(false, ValidationResult::isValid); 32 | assertThat(result.getIssues().size()).isEqualTo(3); 33 | 34 | List issues = result.getIssues(); 35 | 36 | assertThat("url argument is missing").isIn(issues); 37 | assertThat("username argument is missing").isIn(issues); 38 | assertThat("password argument is missing").isIn(issues); 39 | 40 | arg = new String[]{"--user=hello"}; 41 | 42 | result = validator.validate(new DefaultApplicationArguments(arg)); 43 | assertThat(result).returns(false, ValidationResult::isValid); 44 | assertThat(result.getIssues().size()).isEqualTo(2); 45 | issues = result.getIssues(); 46 | 47 | assertThat("url argument is missing").isIn(issues); 48 | assertThat("password argument is missing").isIn(issues); 49 | 50 | arg = new String[]{"--user=hello", "--pass=password", "--hello=world"}; 51 | result = validator.validate(new DefaultApplicationArguments(arg)); 52 | assertThat(result).returns(false, ValidationResult::isValid); 53 | assertThat(result.getIssues().size()).isEqualTo(1); 54 | issues = result.getIssues(); 55 | 56 | assertThat("url argument is missing").isIn(issues); 57 | } 58 | 59 | @Test 60 | void shouldReportEveryDuplicatedEntries() { 61 | String[] arg = new String[]{"--user=hello", "--user=world", "--pass=hi", "--url=333"}; 62 | ValidationResult result = validator.validate(new DefaultApplicationArguments(arg)); 63 | 64 | assertThat(result).returns(false, ValidationResult::isValid); 65 | assertThat(result.getIssues().size()).isEqualTo(1); 66 | 67 | List issues = result.getIssues(); 68 | assertThat("username argument has duplicated entries").isIn(issues); 69 | 70 | arg = new String[]{"--user=hello", "--user=world", "--url=what", "--url=where", "--pass=eee"}; 71 | result = validator.validate(new DefaultApplicationArguments(arg)); 72 | 73 | assertThat(result).returns(false, ValidationResult::isValid); 74 | assertThat(result.getIssues().size()).isEqualTo(2); 75 | 76 | issues = result.getIssues(); 77 | assertThat("username argument has duplicated entries").isIn(issues); 78 | assertThat("url argument has duplicated entries").isIn(issues); 79 | } 80 | 81 | @Test 82 | void shouldReportAsBlankIfEverythingIsPresent() { 83 | String[] arg = 84 | new String[]{"--user=hello", "--pass=pass", "--url=jdbc:mysql://localhost:3306/something"}; 85 | ValidationResult result = validator.validate(new DefaultApplicationArguments(arg)); 86 | assertThat(result).returns(false, ValidationResult::isValid); 87 | assertThat(result.getIssues()) 88 | .hasSize(1) 89 | .contains("database product \"mysql\" is not supported"); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/argument/validator/MappedValueValidatorUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.argument.validator; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | 5 | import io.dsub.discogs.batch.argument.ArgType; 6 | import org.junit.jupiter.api.Test; 7 | import org.junit.jupiter.params.ParameterizedTest; 8 | import org.junit.jupiter.params.provider.EnumSource; 9 | import org.springframework.boot.ApplicationArguments; 10 | import org.springframework.boot.DefaultApplicationArguments; 11 | 12 | class MappedValueValidatorUnitTest { 13 | 14 | final ArgumentValidator validator = new MappedValueValidator(); 15 | 16 | @Test 17 | void validate() { 18 | String chunkSize = "--chunkSize=100"; 19 | ApplicationArguments args = new DefaultApplicationArguments(chunkSize); 20 | ValidationResult result = validator.validate(args); 21 | assertThat(result.isValid()).isTrue(); 22 | } 23 | 24 | @ParameterizedTest 25 | @EnumSource(ArgType.class) 26 | void shouldReportForNullValue(ArgType argType) { 27 | ApplicationArguments args = new DefaultApplicationArguments( 28 | "--" + argType.getGlobalName() + "="); 29 | ValidationResult result = validator.validate(args); 30 | 31 | if (argType.getMinValuesCount() == 0) { 32 | assertThat(result.getIssues().size()).isEqualTo(0); 33 | } else { 34 | assertThat(result.getIssues().size()).isEqualTo(1); 35 | assertThat(result.getIssues().get(0)) 36 | .isEqualTo("missing value for " + argType.getGlobalName()); 37 | } 38 | } 39 | 40 | @ParameterizedTest 41 | @EnumSource(ArgType.class) 42 | void shouldReportNonSupportedMultipleValues(ArgType argType) { 43 | Class supportedType = argType.getSupportedType(); 44 | StringBuilder argBuilder = new StringBuilder("--").append(argType.getGlobalName()).append("="); 45 | 46 | if (supportedType.equals(String.class)) { 47 | argBuilder.append("hello,world,something"); 48 | } else if (supportedType.equals(Long.class)) { 49 | argBuilder.append("333,22,44"); 50 | } 51 | 52 | ApplicationArguments args = new DefaultApplicationArguments(argBuilder.toString()); 53 | ValidationResult result = validator.validate(args); 54 | 55 | int min = argType.getMinValuesCount(); 56 | int max = argType.getMaxValuesCount(); 57 | if (max < 3 || min > 3) { 58 | if (min == max) { 59 | assertThat(result.getIssues().get(0)) 60 | .isEqualTo(argType.getGlobalName() + " expected " + min + " items but got 3 item"); 61 | } else { 62 | assertThat(result.getIssues().get(0)) 63 | .isEqualTo( 64 | argType.getGlobalName() 65 | + " expected " 66 | + min 67 | + " to " 68 | + max 69 | + " items but got 3 item"); 70 | } 71 | } 72 | } 73 | 74 | @ParameterizedTest 75 | @EnumSource(ArgType.class) 76 | void shouldReportUnsupportedValueType(ArgType argType) { 77 | Class supportedType = argType.getSupportedType(); 78 | if (supportedType.equals(String.class)) { 79 | return; 80 | } 81 | 82 | StringBuilder argBuilder = new StringBuilder() 83 | .append("--"); 84 | 85 | int requiredCount = argType.getMinValuesCount(); 86 | 87 | String value = "some...string"; 88 | 89 | for (int i = 0; i < requiredCount; i++) { 90 | if (i == 0) { 91 | argBuilder.append(argType.getGlobalName()).append("="); 92 | } 93 | argBuilder.append(value); 94 | if (i < requiredCount - 1) { 95 | argBuilder.append(","); 96 | } 97 | } 98 | 99 | ApplicationArguments args = new DefaultApplicationArguments(argBuilder.toString()); 100 | ValidationResult result = validator.validate(args); 101 | 102 | String expectedMsg = 103 | "invalid type for " 104 | + argType.getGlobalName() 105 | + ". supported = " 106 | + supportedType.getSimpleName(); 107 | assertThat(result.getIssues().size()).isEqualTo(1); 108 | assertThat(result.getIssues().get(0)).isEqualTo(expectedMsg); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/util/DiscogsJobParametersConverterUnitTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 4 | import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; 5 | import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 6 | 7 | import io.dsub.discogs.batch.exception.InvalidArgumentException; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.Locale; 11 | import java.util.Properties; 12 | import org.junit.jupiter.api.Test; 13 | import org.junit.jupiter.params.ParameterizedTest; 14 | import org.junit.jupiter.params.provider.ValueSource; 15 | import org.springframework.batch.core.JobParameters; 16 | import org.springframework.batch.core.converter.DefaultJobParametersConverter; 17 | import org.springframework.batch.core.converter.JobParametersConverter; 18 | import org.springframework.boot.ApplicationArguments; 19 | import org.springframework.boot.DefaultApplicationArguments; 20 | import org.springframework.integration.support.PropertiesBuilder; 21 | 22 | class DiscogsJobParametersConverterUnitTest { 23 | 24 | final JobParametersConverter delegate = new DefaultJobParametersConverter(); 25 | final DiscogsJobParametersConverter converter = new DiscogsJobParametersConverter(delegate); 26 | 27 | @Test 28 | void getJobParametersBy__ApplicationArguments__ShouldHaveProperValueMapped() 29 | throws InvalidArgumentException { 30 | String[] args = 31 | "url=localhost user=root pass=pass --type=hello,world,java,land --chunk=1000".split(" "); 32 | ApplicationArguments applicationArguments = new DefaultApplicationArguments(args); 33 | JobParameters jobParameters = converter.getJobParameters(applicationArguments); 34 | assertDoesNotThrow(() -> jobParameters.getLong("chunkSize")); 35 | assertThat(jobParameters.getString("url")).isEqualTo("localhost"); 36 | assertThat(jobParameters.getString("username")).isEqualTo("root"); 37 | assertThat(jobParameters.getString("password")).isEqualTo("pass"); 38 | assertThat(jobParameters.getString("type")).isEqualTo("hello,world,java,land"); 39 | assertThat(jobParameters.getLong("chunkSize")).isEqualTo(1000L); 40 | } 41 | 42 | @Test 43 | void afterPropertiesSetMethodShouldThrowIfDelegateIsNull() { 44 | assertThatThrownBy(() -> new DiscogsJobParametersConverter(null).afterPropertiesSet()) 45 | .hasMessage("delegate should not be null"); 46 | } 47 | 48 | @ParameterizedTest 49 | @ValueSource(classes = {String.class, Double.class, Long.class}) 50 | void appendTypeBracketMethodShouldAppendTypesProperly(Class clazz) { 51 | List names = List.of("a", "b", "c"); 52 | names.forEach( 53 | name -> { 54 | String result = converter.appendTypeBracket(name, clazz); 55 | String expected = name + "(" + clazz.getSimpleName().toLowerCase(Locale.ROOT) + ")"; 56 | assertThat(result).isEqualTo(expected); 57 | }); 58 | } 59 | 60 | @Test 61 | void testGetterAndSetterForDelegate() { 62 | JobParametersConverter delegate = new DefaultJobParametersConverter(); 63 | DiscogsJobParametersConverter converter = new DiscogsJobParametersConverter(); 64 | 65 | converter.setDelegate(delegate); // other instance 66 | assertThat(converter.getDelegate()).isEqualTo(delegate); 67 | 68 | converter.setDelegate(this.delegate); // the instance 69 | assertThat(converter.getDelegate()).isNotEqualTo(delegate); 70 | 71 | converter.setDelegate(null); 72 | assertThat(converter.getDelegate()).isNull(); 73 | } 74 | 75 | @Test 76 | void eachDelegatedMethodsShouldResultSameAsItsDelegate() { 77 | String[] args = 78 | "url=localhost user=root pass=pass --type=hello,world,java,land --chunk=1000".split(" "); 79 | PropertiesBuilder builder = new PropertiesBuilder(); 80 | Arrays.stream(args) 81 | .map(arg -> arg.split("=")) 82 | .forEach( 83 | arg -> { 84 | builder.put(arg[0], arg[1]); 85 | }); 86 | Properties properties = builder.get(); 87 | assertThat(converter.getJobParameters(properties)) 88 | .isEqualTo(delegate.getJobParameters(properties)); // vice 89 | JobParameters jobParameters = delegate.getJobParameters(properties); 90 | assertThat(converter.getProperties(jobParameters)) 91 | .isEqualTo(delegate.getProperties(jobParameters)); // versa 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/BatchServiceTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.junit.jupiter.api.Assertions.assertAll; 5 | import static org.junit.jupiter.api.Assertions.fail; 6 | import static org.mockito.ArgumentMatchers.any; 7 | import static org.mockito.Mockito.doCallRealMethod; 8 | import static org.mockito.Mockito.doReturn; 9 | import static org.mockito.Mockito.spy; 10 | import static org.mockito.Mockito.times; 11 | import static org.mockito.Mockito.verify; 12 | 13 | import io.dsub.discogs.batch.argument.handler.ArgumentHandler; 14 | import io.dsub.discogs.batch.argument.validator.ValidationResult; 15 | import io.dsub.discogs.batch.testutil.LogSpy; 16 | import org.junit.jupiter.api.BeforeEach; 17 | import org.junit.jupiter.api.Test; 18 | import org.junit.jupiter.api.extension.RegisterExtension; 19 | import org.mockito.ArgumentCaptor; 20 | import org.mockito.Captor; 21 | import org.mockito.Mock; 22 | import org.mockito.MockedStatic; 23 | import org.mockito.Mockito; 24 | import org.mockito.MockitoAnnotations; 25 | import org.springframework.boot.ApplicationArguments; 26 | import org.springframework.boot.SpringApplication; 27 | import org.springframework.context.ConfigurableApplicationContext; 28 | 29 | class BatchServiceTest { 30 | 31 | BatchService batchService; 32 | 33 | @Mock 34 | ArgumentHandler argumentHandler; 35 | 36 | @Mock 37 | ConfigurableApplicationContext ctx; 38 | 39 | @Captor 40 | ArgumentCaptor argCaptor; 41 | 42 | @RegisterExtension 43 | LogSpy logSpy = new LogSpy(); 44 | 45 | ValidationResult validationResult; 46 | 47 | @BeforeEach 48 | void setUp() { 49 | MockitoAnnotations.openMocks(this); 50 | batchService = spy(new BatchService()); 51 | validationResult = Mockito.mock(ValidationResult.class); 52 | doReturn(argumentHandler).when(batchService).getArgumentHandler(); 53 | doReturn(ctx).when(batchService).runSpringApplication(any()); 54 | doReturn(true).when(validationResult).isValid(); 55 | } 56 | 57 | @Test 58 | void givenArg__WhenRun__ShouldCallHandlerWithSameArg() throws Exception { 59 | // given 60 | String[] args = {"hello", "world"}; 61 | doReturn(args).when(argumentHandler).resolve(any()); 62 | 63 | // when 64 | batchService.run(args); 65 | 66 | // then 67 | assertAll( 68 | () -> verify(argumentHandler, times(2)).resolve(argCaptor.capture()), 69 | () -> assertThat(argCaptor.getValue()).isEqualTo(args) 70 | ); 71 | } 72 | 73 | @Test 74 | void givenArg__WhenRunCalled__ShouldCallHandlerWithSameArg() throws Exception { 75 | // given 76 | String[] args = {"hello", "world"}; 77 | doReturn(args).when(argumentHandler).resolve(argCaptor.capture()); 78 | 79 | // when 80 | batchService.run(args); 81 | 82 | // then 83 | assertAll( 84 | () -> assertThat(argCaptor.getValue()).isNotNull(), 85 | () -> assertThat(argCaptor.getValue()).contains("hello", "world") 86 | ); 87 | } 88 | 89 | @Test 90 | void givenGetArgumentHandler__WhenCalledTwice__ShouldReturnUniqueInstance() { 91 | // given 92 | doCallRealMethod().when(batchService).getArgumentHandler(); 93 | this.argumentHandler = batchService.getArgumentHandler(); 94 | 95 | // when 96 | ArgumentHandler that = batchService.getArgumentHandler(); 97 | 98 | // then 99 | assertThat(this.argumentHandler).isNotEqualTo(that); 100 | } 101 | 102 | @Test 103 | void whenRunCalled__ShouldCallRunSpringApplicationWithSameArgs() { 104 | try (MockedStatic app = Mockito.mockStatic(SpringApplication.class)) { 105 | String[] args = {"hello", "world"}; 106 | 107 | app.when(() -> SpringApplication.run(BatchApplication.class, args)).thenReturn(ctx); 108 | doReturn(args).when(argumentHandler).resolve(args); 109 | doReturn(true).when(validationResult).isValid(); 110 | doCallRealMethod().when(batchService).runSpringApplication(args); 111 | 112 | // when 113 | ConfigurableApplicationContext that = batchService.run(args); 114 | 115 | // then 116 | verify(batchService, times(1)).runSpringApplication(args); 117 | assertThat(this.ctx).isEqualTo(that); 118 | app.verify(() -> SpringApplication.run(BatchApplication.class, args), times(1)); 119 | } catch (Exception e) { 120 | fail(e); 121 | } 122 | } 123 | } -------------------------------------------------------------------------------- /src/main/java/io/dsub/discogs/batch/util/DefaultMalformedDateParser.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch.util; 2 | 3 | import java.time.LocalDate; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | public class DefaultMalformedDateParser implements MalformedDateParser { 8 | 9 | private static final Pattern YEAR_PATTERN = Pattern.compile("^([\\d]{4}).*"); 10 | 11 | private static final Pattern YEAR_PRESENT = Pattern.compile("^([\\w]{4}).*"); 12 | 13 | private static final Pattern MONTH_PATTERN = 14 | Pattern.compile("^[\\w]{2,4}[- /.](0*[1-9]|1[0-2])[- /.]?"); 15 | 16 | private static final Pattern MONTH_PRESENT = Pattern.compile("^[\\w]{2,4}[- /.](0*[\\w]{1,2}).*"); 17 | 18 | private static final Pattern DAY_PATTERN = 19 | Pattern.compile("^[\\d]{4}([- /.])(0*([1-9]|1[0-2]))\\1(0*(3[0-1]|[1-2][0-9]|[1-9]))$"); 20 | 21 | private static final Pattern FLAT_PATTERN = Pattern.compile("^([\\d]{4})([\\d]{2})([\\d]{2})$"); 22 | 23 | @Override 24 | public boolean isMonthValid(String date) { 25 | if (date == null) { 26 | return false; 27 | } 28 | return MONTH_PATTERN.matcher(date).matches(); 29 | } 30 | 31 | @Override 32 | public boolean isYearValid(String date) { 33 | if (date == null) { 34 | return false; 35 | } 36 | return YEAR_PATTERN.matcher(date).matches(); 37 | } 38 | 39 | @Override 40 | public boolean isDayValid(String date) { 41 | if (date == null) { 42 | return false; 43 | } 44 | return parseDay(date) > 0; 45 | } 46 | 47 | @Override 48 | public LocalDate parse(String source) { 49 | if (source == null) { 50 | return null; 51 | } 52 | 53 | String normalized = source.replaceAll("[^\\d._/ -]", "0"); 54 | 55 | if (normalized.isBlank() || normalized.replaceAll("0", "").length() == 0) { 56 | return null; 57 | } 58 | 59 | int year = parseYear(normalized); 60 | if (year < 1) { 61 | return null; 62 | } 63 | 64 | int month = parseMonth(normalized); 65 | 66 | if (month < 0) { 67 | return LocalDate.of(year, 1, 1); 68 | } 69 | 70 | int day = parseDay(normalized); 71 | 72 | if (day < 0) { 73 | return LocalDate.of(year, month, 1); 74 | } 75 | 76 | return LocalDate.of(year, month, day); 77 | } 78 | 79 | private int parseDay(String date) { 80 | Matcher flatMatcher = FLAT_PATTERN.matcher(date); 81 | Matcher dayMatcher = DAY_PATTERN.matcher(date); 82 | if (!flatMatcher.matches() && !dayMatcher.matches()) { 83 | return -1; 84 | } 85 | 86 | int year = parseYear(date); 87 | int month = parseMonth(date); 88 | if (year < 0 || month < 0) { 89 | return -1; 90 | } 91 | int maxDay = getMaxDayOfMonth(year, month); 92 | 93 | String possibleDay; 94 | 95 | if (flatMatcher.matches()) { 96 | possibleDay = flatMatcher.group(3); 97 | } else { 98 | possibleDay = dayMatcher.group(4); 99 | } 100 | 101 | int day = Integer.parseInt(possibleDay); 102 | 103 | return day <= 0 || day > maxDay ? -1 : day; 104 | } 105 | 106 | private int parseYear(String date) { 107 | Matcher matcher = FLAT_PATTERN.matcher(date); 108 | if (matcher.matches()) { 109 | int year = Integer.parseInt(matcher.group(1)); 110 | return year > 1000 && year < LocalDate.now().getYear() + 1 ? year : -1; 111 | } 112 | matcher = YEAR_PATTERN.matcher(date); 113 | if (matcher.matches()) { 114 | String possibleYear = matcher.group(1); 115 | int year = Integer.parseInt(possibleYear); 116 | if (year > 1000 && year < LocalDate.now().getYear() + 1) { 117 | return year; 118 | } 119 | } 120 | return -1; 121 | } 122 | 123 | private int parseMonth(String date) { 124 | Matcher matcher = FLAT_PATTERN.matcher(date); 125 | if (matcher.matches()) { 126 | int month = Integer.parseInt(matcher.group(2)); 127 | if (month > 0 && month < 13) { 128 | return month; 129 | } 130 | } 131 | matcher = MONTH_PRESENT.matcher(date); 132 | if (matcher.matches()) { 133 | String possibleMonth = matcher.group(1); 134 | int monthValue = Integer.parseInt(possibleMonth); 135 | if (monthValue > 0 && monthValue < 13) { 136 | return monthValue; 137 | } 138 | } 139 | return -1; 140 | } 141 | 142 | private int getMaxDayOfMonth(int year, int month) { 143 | return LocalDate.of(year, month, 1).lengthOfMonth(); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/test/java/io/dsub/discogs/batch/JobLaunchingRunnerTest.java: -------------------------------------------------------------------------------- 1 | package io.dsub.discogs.batch; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.junit.jupiter.api.Assertions.fail; 5 | import static org.mockito.Mockito.doReturn; 6 | import static org.mockito.Mockito.spy; 7 | import static org.mockito.Mockito.times; 8 | import static org.mockito.Mockito.verify; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Collections; 12 | import java.util.List; 13 | import java.util.concurrent.CountDownLatch; 14 | import org.junit.jupiter.api.BeforeEach; 15 | import org.junit.jupiter.api.Test; 16 | import org.junit.jupiter.params.ParameterizedTest; 17 | import org.junit.jupiter.params.provider.ValueSource; 18 | import org.mockito.InjectMocks; 19 | import org.mockito.Mock; 20 | import org.mockito.MockedStatic; 21 | import org.mockito.Mockito; 22 | import org.mockito.MockitoAnnotations; 23 | import org.springframework.batch.core.ExitStatus; 24 | import org.springframework.batch.core.Job; 25 | import org.springframework.batch.core.JobExecution; 26 | import org.springframework.batch.core.JobParameters; 27 | import org.springframework.batch.core.JobParametersInvalidException; 28 | import org.springframework.batch.core.launch.JobLauncher; 29 | import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; 30 | import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; 31 | import org.springframework.batch.core.repository.JobRestartException; 32 | import org.springframework.boot.ApplicationArguments; 33 | import org.springframework.boot.ExitCodeGenerator; 34 | import org.springframework.boot.SpringApplication; 35 | import org.springframework.context.ConfigurableApplicationContext; 36 | 37 | class JobLaunchingRunnerTest { 38 | 39 | @Mock 40 | Job job; 41 | @Mock 42 | JobParameters discogsJobParameters; 43 | @Mock 44 | JobLauncher jobLauncher; 45 | @Mock 46 | ApplicationArguments args; 47 | @Mock 48 | JobExecution jobExecution; 49 | @Mock 50 | ExitStatus exitStatus; 51 | @Mock 52 | ConfigurableApplicationContext context; 53 | @Mock 54 | CountDownLatch countDownLatch; 55 | @InjectMocks 56 | JobLaunchingRunner runner; 57 | 58 | @BeforeEach 59 | void setUp() 60 | throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, 61 | JobParametersInvalidException, JobRestartException { 62 | MockitoAnnotations.openMocks(this); 63 | doReturn(jobExecution).when(jobLauncher).run(job, discogsJobParameters); 64 | doReturn(false).when(exitStatus).isRunning(); 65 | doReturn(false).when(jobExecution).isRunning(); 66 | doReturn(exitStatus).when(jobExecution).getExitStatus(); 67 | doReturn(Collections.emptyList()).when(jobExecution).getFailureExceptions(); 68 | runner = spy(runner); 69 | } 70 | 71 | @Test 72 | void whenRunCalled__ShouldCallJobLauncherWithJobAndJobParameters() throws Exception { 73 | // when 74 | runner.run(args); 75 | 76 | // then 77 | verify(jobLauncher, times(1)).run(job, discogsJobParameters); 78 | } 79 | 80 | @Test 81 | void whenRunCalled__ShouldCallSpringApplicationExit() { 82 | try (MockedStatic app = Mockito.mockStatic(SpringApplication.class)) { 83 | // given 84 | ExitCodeGenerator exitCodeGenerator = runner.getExitCodeGenerator(jobExecution); 85 | doReturn(exitCodeGenerator).when(runner).getExitCodeGenerator(jobExecution); 86 | 87 | // when 88 | runner.run(args); 89 | 90 | // then 91 | app.verify(() -> SpringApplication.exit(context, exitCodeGenerator), times(1)); 92 | 93 | } catch (Exception e) { 94 | fail(e); 95 | } 96 | } 97 | 98 | @Test 99 | void whenRunCalled__ShouldWaitForCountDownLatch() throws Exception { 100 | runner.run(args); 101 | verify(countDownLatch, times(1)).await(); 102 | } 103 | 104 | @ParameterizedTest 105 | @ValueSource(ints = {0, 1}) 106 | void givenJobExecutionHasFailureExceptions__WhenExitCodeGeneratorGetExitCode__ShouldReturnProperExitCode( 107 | int errorsCnt) { 108 | // given 109 | List mockList = spy(new ArrayList<>()); 110 | doReturn(errorsCnt).when(mockList).size(); 111 | doReturn(mockList).when(jobExecution).getFailureExceptions(); 112 | 113 | // when 114 | int exitCode = runner.getExitCodeGenerator(jobExecution).getExitCode(); 115 | 116 | // then 117 | assertThat(exitCode).isEqualTo(errorsCnt > 0 ? 1 : 0); 118 | } 119 | } 120 | --------------------------------------------------------------------------------