├── .github └── PULL_REQUEST_TEMPLATE.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── amazon │ └── pocketEtl │ ├── EtlCombineStage.java │ ├── EtlConsumerStage.java │ ├── EtlExtractStage.java │ ├── EtlLoadStage.java │ ├── EtlMetrics.java │ ├── EtlProducerStage.java │ ├── EtlProfilingScope.java │ ├── EtlRunner.java │ ├── EtlStageChain.java │ ├── EtlStream.java │ ├── EtlTransformStage.java │ ├── Extractor.java │ ├── Loader.java │ ├── Transformer.java │ ├── common │ └── ThrowingFunction.java │ ├── core │ ├── DefaultLoggingStrategy.java │ ├── EtlStreamObject.java │ ├── consumer │ │ ├── EtlConsumer.java │ │ ├── EtlConsumerFactory.java │ │ ├── ExecutorEtlConsumer.java │ │ ├── LoaderEtlConsumer.java │ │ ├── LogAsErrorEtlConsumer.java │ │ ├── MetricsEmissionEtlConsumer.java │ │ ├── SmartEtlConsumer.java │ │ └── TransformerEtlConsumer.java │ ├── executor │ │ ├── EtlExecutor.java │ │ ├── EtlExecutorFactory.java │ │ ├── ExecutorServiceEtlExecutor.java │ │ └── ImmediateExecutionEtlExecutor.java │ └── producer │ │ ├── EtlProducer.java │ │ ├── EtlProducerFactory.java │ │ ├── ExecutorEtlProducer.java │ │ └── ExtractorEtlProducer.java │ ├── exception │ ├── DependencyException.java │ ├── GenericEtlException.java │ └── UnrecoverableStreamFailureException.java │ ├── extractor │ ├── CsvInputStreamMapper.java │ ├── InputStreamExtractor.java │ ├── InputStreamMapper.java │ ├── IterableExtractor.java │ ├── IteratorExtractor.java │ ├── JSONStringMapper.java │ ├── S3BufferedExtractor.java │ ├── S3BufferedInputStream.java │ ├── SqlExtractor.java │ ├── SqsExtractor.java │ ├── WrappedExtractor.java │ └── WrappedInputStream.java │ ├── integration │ ├── RedshiftJdbcClient.java │ └── db │ │ ├── SqlStringHelpers.java │ │ └── jdbi │ │ ├── EtlBeanMapper.java │ │ ├── EtlBeanMapperFactory.java │ │ ├── EtlJdbi.java │ │ └── PostgresStringArrayArgumentFactory.java │ ├── loader │ ├── CsvStringSerializer.java │ ├── DynamoDbLoader.java │ ├── MetricsLoader.java │ ├── ParallelLoader.java │ ├── RedshiftBulkLoader.java │ ├── RedshiftLoadStrategy.java │ ├── S3FastLoader.java │ ├── S3PartFileKeyGenerator.java │ ├── StringSerializer.java │ └── WrappedLoader.java │ ├── lookup │ ├── CachingLoaderLookup.java │ ├── Lookup.java │ └── SemaphoreFactory.java │ └── transformer │ ├── FilterTransformer.java │ ├── MapTransformer.java │ └── filter │ └── ContainsFilter.java └── test └── java ├── com └── amazon │ └── pocketEtl │ ├── EtlCombineStageTest.java │ ├── EtlExtractStageTest.java │ ├── EtlLoadStageTest.java │ ├── EtlProfilingScopeTest.java │ ├── EtlStreamTest.java │ ├── EtlTestBase.java │ ├── EtlTransformStageTest.java │ ├── core │ ├── EtlStreamObjectTest.java │ ├── consumer │ │ ├── EtlConsumerFactoryTest.java │ │ ├── ExecutorEtlConsumerTest.java │ │ ├── LoaderEtlConsumerTest.java │ │ ├── LogAsErrorEtlConsumerTest.java │ │ ├── MetricsEmissionEtlConsumerTest.java │ │ ├── SmartEtlConsumerTest.java │ │ └── TransformerEtlConsumerTest.java │ ├── executor │ │ ├── EtlExecutorFactoryBlockingFixedThreadsTest.java │ │ ├── EtlExecutorFactoryUnboundFixedThreadsTest.java │ │ ├── ExecutorServiceEtlExecutorTest.java │ │ └── ImmediateExecutionEtlExecutorTest.java │ └── producer │ │ ├── EtlProducerFactoryTest.java │ │ ├── ExecutorEtlProducerTest.java │ │ └── ExtractorEtlProducerTest.java │ ├── extractor │ ├── CsvInputStreamMapperTest.java │ ├── InputStreamExtractorTest.java │ ├── IterableExtractorTest.java │ ├── IteratorExtractorTest.java │ ├── JSONStringMapperTest.java │ ├── S3BufferedExtractorTest.java │ ├── S3BufferedInputStreamTest.java │ ├── SqlExtractorTest.java │ ├── SqsExtractorTest.java │ └── TestDTO.java │ ├── integration │ ├── RedshiftJdbcClientTest.java │ └── db │ │ ├── SqlStringHelpersTest.java │ │ └── jdbi │ │ └── EtlBeanMapperTest.java │ ├── loader │ ├── CsvStringSerializerTest.java │ ├── DynamoDbLoaderTest.java │ ├── MetricsLoaderTest.java │ ├── ParallelLoaderTest.java │ ├── RedshiftBulkLoaderTest.java │ └── S3FastLoaderTest.java │ ├── lookup │ └── CachingLoaderLookupTest.java │ └── transformer │ ├── FilterTransformerTest.java │ ├── MapTransformerTest.java │ └── filter │ └── ContainsFilterTest.java └── functionalTests ├── BufferExtractor.java ├── BufferLoader.java ├── DynamoDbFunctionalTest.java ├── FilterFunctionalTest.java ├── ImmutabilityTest.java ├── LoggingStrategyFunctionalTest.java ├── MockedS3FunctionalTest.java ├── SimpleEtlFunctionalTest.java ├── SimpleFluentFunctionalTest.java ├── SqlExtractorFunctionalTest.java ├── StreamFailureTest.java ├── TestDTO.java ├── TestDTO2.java ├── ThrowsEtlConsumer.java └── TransformerFanoutTest.java /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/awslabs/pocket-etl/issues), or [recently closed](https://github.com/awslabs/pocket-etl/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/pocket-etl/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/awslabs/pocket-etl/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Pocket Etl 2 | Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlCombineStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import java.util.Collection; 19 | import java.util.stream.Collectors; 20 | 21 | import javax.annotation.Nonnull; 22 | 23 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 24 | import com.amazon.pocketEtl.core.producer.EtlProducer; 25 | 26 | import lombok.AccessLevel; 27 | import lombok.Getter; 28 | 29 | @Getter(AccessLevel.PACKAGE) 30 | class EtlCombineStage extends EtlProducerStage { 31 | private final static String DEFAULT_COMBINE_STAGE_NAME = "EtlStream.Combine"; 32 | 33 | private final Collection stageChains; 34 | 35 | private EtlCombineStage(@Nonnull Collection stageChains, @Nonnull String stageName) { 36 | super(stageName); 37 | 38 | if (stageChains.size() < 2) { 39 | throw new IllegalArgumentException("A combined EtlStream must be constructed with at least two component streams."); 40 | } 41 | 42 | this.stageChains = stageChains; 43 | } 44 | 45 | static EtlCombineStage of(@Nonnull Collection streamsToCombine) { 46 | Collection stageChains = streamsToCombine.stream().map(EtlStream::getStageChain).collect(Collectors.toList()); 47 | return new EtlCombineStage(stageChains, DEFAULT_COMBINE_STAGE_NAME); 48 | } 49 | 50 | @Override 51 | public EtlCombineStage withName(@Nonnull String stageName) { 52 | return new EtlCombineStage(getStageChains(), stageName); 53 | } 54 | 55 | @Override 56 | Collection constructProducersForStage(EtlConsumer downstreamConsumer) { 57 | return getStageChains().stream() 58 | .map(stageChain -> stageChain.constructComponentProducers(downstreamConsumer)) 59 | .flatMap(Collection::stream) 60 | .collect(Collectors.toList()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlExtractStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import java.util.Collection; 19 | import java.util.Collections; 20 | import java.util.stream.Collectors; 21 | 22 | import javax.annotation.Nonnull; 23 | 24 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 25 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 26 | import com.amazon.pocketEtl.core.producer.EtlProducer; 27 | import com.amazon.pocketEtl.core.producer.EtlProducerFactory; 28 | 29 | import lombok.AccessLevel; 30 | import lombok.Getter; 31 | 32 | @Getter(AccessLevel.PACKAGE) 33 | class EtlExtractStage extends EtlProducerStage { 34 | private final static String DEFAULT_EXTRACT_STAGE_NAME = "EtlStream.Extract"; 35 | private final static EtlExecutorFactory defaultExecutorFactory = new EtlExecutorFactory(); 36 | private final static EtlProducerFactory defaultProducerFactory = new EtlProducerFactory(defaultExecutorFactory); 37 | 38 | private final Collection> extractors; 39 | private final EtlProducerFactory etlProducerFactory; 40 | 41 | EtlExtractStage(@Nonnull Collection> extractors, 42 | @Nonnull String stageName, 43 | @Nonnull EtlProducerFactory etlProducerFactory) { 44 | super(stageName); 45 | 46 | if (extractors.isEmpty()) { 47 | throw new IllegalArgumentException("An EtlStream must be constructed with at least one extractor."); 48 | } 49 | 50 | this.extractors = extractors; 51 | this.etlProducerFactory = etlProducerFactory; 52 | } 53 | 54 | @Override 55 | public EtlExtractStage withName(@Nonnull String stageName) { 56 | return new EtlExtractStage(getExtractors(), stageName, getEtlProducerFactory()); 57 | } 58 | 59 | static EtlExtractStage of(@Nonnull Extractor extractor) { 60 | return new EtlExtractStage(Collections.singletonList(extractor), DEFAULT_EXTRACT_STAGE_NAME, defaultProducerFactory); 61 | } 62 | 63 | static EtlExtractStage of(@Nonnull Collection> extractors) { 64 | return new EtlExtractStage(extractors, DEFAULT_EXTRACT_STAGE_NAME, defaultProducerFactory); 65 | } 66 | 67 | @Override 68 | Collection constructProducersForStage(EtlConsumer downstreamConsumer) { 69 | return extractors.stream().map(extractor -> 70 | getEtlProducerFactory().newExtractorProducer(getStageName(), extractor, downstreamConsumer)) 71 | .collect(Collectors.toList()); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlLoadStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import static org.apache.logging.log4j.LogManager.getLogger; 19 | 20 | import java.util.function.Function; 21 | 22 | import javax.annotation.Nonnull; 23 | 24 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 25 | import com.amazon.pocketEtl.core.consumer.EtlConsumerFactory; 26 | import com.amazon.pocketEtl.core.executor.EtlExecutor; 27 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 28 | 29 | import lombok.AccessLevel; 30 | import lombok.Getter; 31 | 32 | @Getter(AccessLevel.PACKAGE) 33 | class EtlLoadStage extends EtlConsumerStage { 34 | private final static String DEFAULT_LOAD_STAGE_NAME = "EtlStream.Load"; 35 | 36 | private final static EtlExecutorFactory defaultExecutorFactory = new EtlExecutorFactory(); 37 | private final static EtlConsumerFactory defaultConsumerFactory = new EtlConsumerFactory(defaultExecutorFactory); 38 | 39 | private final Loader loader; 40 | private final EtlExecutorFactory etlExecutorFactory; 41 | private final EtlConsumerFactory etlConsumerFactory; 42 | 43 | EtlLoadStage(@Nonnull Class classForStage, 44 | @Nonnull Loader loader, 45 | @Nonnull String stageName, 46 | @Nonnull Integer numberOfThreads, 47 | @Nonnull Function objectLogger, 48 | @Nonnull EtlExecutorFactory etlExecutorFactory, 49 | @Nonnull EtlConsumerFactory etlConsumerFactory) { 50 | super(classForStage, stageName, numberOfThreads, objectLogger); 51 | this.loader = loader; 52 | this.etlExecutorFactory = etlExecutorFactory; 53 | this.etlConsumerFactory = etlConsumerFactory; 54 | } 55 | 56 | @Override 57 | public EtlLoadStage withObjectLogger(@Nonnull Function objectLogger) { 58 | return new EtlLoadStage<>(getClassForStage(), getLoader(), getStageName(), getNumberOfThreads(), objectLogger, 59 | getEtlExecutorFactory(), getEtlConsumerFactory()); 60 | } 61 | 62 | @Override 63 | public EtlLoadStage withName(@Nonnull String stageName) { 64 | return new EtlLoadStage<>(getClassForStage(), getLoader(), stageName, getNumberOfThreads(), getObjectLogger(), 65 | getEtlExecutorFactory(), getEtlConsumerFactory()); 66 | } 67 | 68 | @Override 69 | public EtlLoadStage withThreads(@Nonnull Integer threads) { 70 | return new EtlLoadStage<>(getClassForStage(), getLoader(), getStageName(), threads, getObjectLogger(), 71 | getEtlExecutorFactory(), getEtlConsumerFactory()); 72 | } 73 | 74 | static EtlLoadStage of(@Nonnull Class classForStage, @Nonnull Loader loader) { 75 | return new EtlLoadStage<>(classForStage, loader, DEFAULT_LOAD_STAGE_NAME, getDefaultNumberOfWorkers(), 76 | getDefaultObjectLogger(), defaultExecutorFactory, defaultConsumerFactory); 77 | } 78 | 79 | @Override 80 | EtlConsumer constructConsumerForStage(EtlConsumer downstreamConsumer) { 81 | EtlExecutor stageExecutor = getEtlExecutorFactory().newBlockingFixedThreadsEtlExecutor(getNumberOfThreads(), 82 | getDefaultQueueSize()); 83 | EtlConsumer errorConsumer = getEtlConsumerFactory().newLogAsErrorConsumer(getStageName(), getLogger(getStageName()), 84 | getClassForStage(), getObjectLogger()); 85 | 86 | return getEtlConsumerFactory().newLoader(getStageName(), getLoader(), getClassForStage(), errorConsumer, stageExecutor); 87 | } 88 | 89 | @Override 90 | boolean isTerminal() { 91 | return true; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import java.io.Closeable; 19 | 20 | /** 21 | * Interface for an object to record counters and timers for metrics and profiling. Clients wishing to get profiling 22 | * data from their PocketETL jobs should write an implementation for this interface or an adapter to a metrics library 23 | * of their choice. 24 | */ 25 | public interface EtlMetrics extends Closeable { 26 | /** 27 | * Create a child metrics object that remains linked to this one. When the child metrics object is closed, all of its 28 | * data rolls up into the parent metrics object that created it. A new child metrics object would thus be created 29 | * every time a new profiling scope was entered. 30 | * @return A new EtlMetrics object that is a linked child to the current EtlMetrics object. 31 | */ 32 | EtlMetrics createChildMetrics(); 33 | 34 | /** 35 | * Adds a count to the metrics object. Aggregation strategies can vary depending on implementation. 36 | * @param keyName The metrics key. 37 | * @param valueInUnits The value of the count. 38 | */ 39 | void addCount(String keyName, double valueInUnits); 40 | 41 | /** 42 | * Adds a time to the metrics object. Aggregation strategies can vary depending on implementation. 43 | * @param keyName The metrics key. 44 | * @param valueInMilliSeconds The value of the timer in milliseconds. 45 | */ 46 | void addTime(String keyName, double valueInMilliSeconds); 47 | 48 | /** 49 | * Closes the metrics object and if linked to a parent metrics object will roll-up all timers and counts to the 50 | * parent metrics object. 51 | */ 52 | @Override 53 | void close(); 54 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlProducerStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import java.util.Arrays; 19 | import java.util.Collection; 20 | 21 | import javax.annotation.Nonnull; 22 | 23 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 24 | import com.amazon.pocketEtl.core.producer.EtlProducer; 25 | 26 | import lombok.AccessLevel; 27 | import lombok.Getter; 28 | import lombok.RequiredArgsConstructor; 29 | 30 | /** 31 | * Abstract class that represents the producer stage in an EtlStream. An EtlStream consists of a producer stage chained 32 | * to one or more consumer stages. 33 | */ 34 | @Getter(AccessLevel.PACKAGE) 35 | @RequiredArgsConstructor 36 | public abstract class EtlProducerStage { 37 | /** 38 | * Static constructor for an EtlProducerStage that extracts data and puts it on the stream. Used as a component in an 39 | * EtlStream. 40 | * @param extractor An extractor that extracts data from somewhere to be put on the stream. 41 | * @return An EtlProducerStage that can be used as a component for an EtlStream. 42 | */ 43 | public static EtlProducerStage extract(@Nonnull Extractor extractor) { 44 | return EtlExtractStage.of(extractor); 45 | } 46 | 47 | /** 48 | * Static constructor for an EtlProducerStage that extracts data from multiple sources and puts it on the stream. 49 | * Used as a component in an EtlStream. The extractors will run in parallel when the stream is run. 50 | * @param extractors A collection of extractors that extract data from somewhere to be put on the stream. 51 | * @return An EtlProducerStage that can be used as a component for an EtlStream. 52 | */ 53 | public static EtlProducerStage extract(@Nonnull Extractor ...extractors) { 54 | return EtlExtractStage.of(Arrays.asList(extractors)); 55 | } 56 | 57 | /** 58 | * Static constructor for an EtlProducerStage that extracts data from multiple sources and puts it on the stream. 59 | * Used as a component in an EtlStream. The extractors will run in parallel when the stream is run. 60 | * @param extractors A collection of extractors that extract data from somewhere to be put on the stream. 61 | * @return An EtlProducerStage that can be used as a component for an EtlStream. 62 | */ 63 | public static EtlProducerStage extract(@Nonnull Collection> extractors) { 64 | return EtlExtractStage.of(extractors); 65 | } 66 | 67 | /** 68 | * Static constructor for an EtlProducerStage that is composed from other EtlStream objects. The combined streams 69 | * will run as a black-boxed producer in its own right in the stream and if any of those streams are not terminated, 70 | * data will flow out of the stage into other consumer stages attached to this one. 71 | * @param streamsToCombine A collection of EtlStream objects to combine into a single stage of another stream. 72 | * @return An EtlProducerStage that can be used as a component for an EtlStream. 73 | */ 74 | public static EtlProducerStage combine(@Nonnull Collection streamsToCombine) { 75 | return EtlCombineStage.of(streamsToCombine); 76 | } 77 | 78 | /** 79 | * Construct a new EtlProducerStage object that is the copy of an existing one but with a new specific value. 80 | * @param stageName This is a human readable name for the stage that is used in profiling and logging and will help you 81 | * build specific monitors and alarms for specific jobs or stages of those jobs. 82 | * @return A new EtlProducerStage object. 83 | */ 84 | public abstract EtlProducerStage withName(@Nonnull String stageName); 85 | 86 | /***************************************************************************************************************/ 87 | 88 | private final String stageName; 89 | 90 | abstract Collection constructProducersForStage(EtlConsumer downstreamConsumer); 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import com.amazon.pocketEtl.core.producer.EtlProducer; 19 | 20 | /** 21 | * Internal functional interface for a method that runs an ETL job. Used for test injection. 22 | */ 23 | @FunctionalInterface 24 | interface EtlRunner { 25 | void run(EtlProducer producer, EtlMetrics parentMetrics) throws Exception; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlStageChain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 19 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 20 | import com.amazon.pocketEtl.core.producer.EtlProducer; 21 | import com.amazon.pocketEtl.core.producer.EtlProducerFactory; 22 | import com.google.common.collect.ImmutableList; 23 | import lombok.Getter; 24 | 25 | import javax.annotation.Nullable; 26 | import java.util.Collection; 27 | import java.util.concurrent.atomic.AtomicReference; 28 | 29 | @Getter 30 | class EtlStageChain { 31 | private final static String DEFAULT_COMBINE_STAGE_NAME = "EtlStream.Combine"; 32 | 33 | private final EtlExecutorFactory etlExecutorFactory = new EtlExecutorFactory(); 34 | private final EtlProducerFactory etlProducerFactory = new EtlProducerFactory(etlExecutorFactory); 35 | private final ImmutableList consumerStagesStack; 36 | private final EtlProducerStage headStage; 37 | 38 | EtlStageChain(EtlStageChain priorChain, EtlConsumerStage newConsumerStage) { 39 | consumerStagesStack = ImmutableList.builder() 40 | .add(newConsumerStage) 41 | .addAll(priorChain.getConsumerStagesStack()) 42 | .build(); 43 | 44 | this.headStage = priorChain.getHeadStage(); 45 | } 46 | 47 | EtlStageChain(EtlProducerStage headStage) { 48 | consumerStagesStack = ImmutableList.of(); 49 | this.headStage = headStage; 50 | } 51 | 52 | @Nullable 53 | private EtlConsumer constructConsumerChain(@Nullable EtlConsumer downstreamConsumer) { 54 | // Although forEachRemaining in this context is guaranteed to execute in series, java 8 will not allow 55 | // overwriting a non-final variable from within a closure so a final AtomicReference is used to wrap the dynamic 56 | // value. 57 | AtomicReference consumerChainHead = new AtomicReference<>(downstreamConsumer); 58 | 59 | getConsumerStagesStack().iterator().forEachRemaining(stage -> 60 | consumerChainHead.set(stage.constructConsumerForStage(consumerChainHead.get()))); 61 | 62 | return consumerChainHead.get(); 63 | } 64 | 65 | EtlProducer constructProducer() { 66 | Collection etlProducers = constructComponentProducers(null); 67 | return etlProducers.size() == 1 ? etlProducers.iterator().next() : 68 | etlProducerFactory.combineProducers(DEFAULT_COMBINE_STAGE_NAME, etlProducers, etlProducers.size()); 69 | } 70 | 71 | Collection constructComponentProducers(@Nullable EtlConsumer downstreamConsumer) { 72 | return getHeadStage().constructProducersForStage(constructConsumerChain(downstreamConsumer)); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/EtlTransformStage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | 19 | import static org.apache.logging.log4j.LogManager.getLogger; 20 | 21 | import java.util.function.Function; 22 | 23 | import javax.annotation.Nonnull; 24 | 25 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 26 | import com.amazon.pocketEtl.core.consumer.EtlConsumerFactory; 27 | import com.amazon.pocketEtl.core.executor.EtlExecutor; 28 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 29 | 30 | import lombok.AccessLevel; 31 | import lombok.Getter; 32 | 33 | @Getter(AccessLevel.PACKAGE) 34 | class EtlTransformStage extends EtlConsumerStage { 35 | private final static String DEFAULT_TRANSFORM_STAGE_NAME = "EtlStream.Transform"; 36 | 37 | private final static EtlExecutorFactory defaultExecutorFactory = new EtlExecutorFactory(); 38 | private final static EtlConsumerFactory defaultConsumerFactory = new EtlConsumerFactory(defaultExecutorFactory); 39 | 40 | private final Transformer transformer; 41 | private final EtlExecutorFactory etlExecutorFactory; 42 | private final EtlConsumerFactory etlConsumerFactory; 43 | 44 | EtlTransformStage(@Nonnull Class classForStage, 45 | @Nonnull Transformer transformer, 46 | @Nonnull String stageName, 47 | @Nonnull Integer numberOfThreads, 48 | @Nonnull Function objectLogger, 49 | @Nonnull EtlExecutorFactory etlExecutorFactory, 50 | @Nonnull EtlConsumerFactory etlConsumerFactory) { 51 | super(classForStage, stageName, numberOfThreads, objectLogger); 52 | this.transformer = transformer; 53 | this.etlExecutorFactory = etlExecutorFactory; 54 | this.etlConsumerFactory = etlConsumerFactory; 55 | } 56 | 57 | @Override 58 | public EtlTransformStage withObjectLogger(@Nonnull Function objectLogger) { 59 | return new EtlTransformStage<>(getClassForStage(), getTransformer(), getStageName(), getNumberOfThreads(), 60 | objectLogger, getEtlExecutorFactory(), getEtlConsumerFactory()); 61 | } 62 | 63 | @Override 64 | public EtlTransformStage withName(@Nonnull String stageName) { 65 | return new EtlTransformStage<>(getClassForStage(), getTransformer(), stageName, getNumberOfThreads(), 66 | getObjectLogger(), getEtlExecutorFactory(), getEtlConsumerFactory()); 67 | } 68 | 69 | @Override 70 | public EtlTransformStage withThreads(@Nonnull Integer threads) { 71 | return new EtlTransformStage<>(getClassForStage(), getTransformer(), getStageName(), threads, getObjectLogger(), 72 | getEtlExecutorFactory(), getEtlConsumerFactory()); 73 | } 74 | 75 | static EtlTransformStage of(@Nonnull Class classForStage, @Nonnull Transformer transformer) { 76 | return new EtlTransformStage<>(classForStage, transformer, DEFAULT_TRANSFORM_STAGE_NAME, 77 | getDefaultNumberOfWorkers(), getDefaultObjectLogger(), defaultExecutorFactory, defaultConsumerFactory); 78 | } 79 | 80 | @Override 81 | EtlConsumer constructConsumerForStage(EtlConsumer downstreamConsumer) { 82 | if (downstreamConsumer == null) { 83 | throw new IllegalArgumentException("Attempt to construct transform stage with null downstream consumer"); 84 | } 85 | 86 | EtlExecutor stageExecutor = getEtlExecutorFactory().newBlockingFixedThreadsEtlExecutor(getNumberOfThreads(), 87 | getDefaultQueueSize()); 88 | EtlConsumer errorConsumer = getEtlConsumerFactory().newLogAsErrorConsumer(getStageName(), getLogger(getStageName()), 89 | getClassForStage(), getObjectLogger()); 90 | 91 | return getEtlConsumerFactory().newTransformer(getStageName(), getTransformer(), getClassForStage(), 92 | downstreamConsumer, errorConsumer, stageExecutor); 93 | } 94 | 95 | @Override 96 | boolean isTerminal() { 97 | return false; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/Extractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import javax.annotation.Nullable; 19 | import java.util.Optional; 20 | 21 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 22 | 23 | /** 24 | * Interface for an extractor object that either extracts objects from some kind of storage. 25 | * 26 | * @param Type of object that is extracted. 27 | */ 28 | @FunctionalInterface 29 | public interface Extractor extends AutoCloseable { 30 | /** 31 | * Attempts to extract the next object from the backing store. 32 | * 33 | * @return An optional object that contains the next extracted object or empty if there are no objects left to be 34 | * extracted. 35 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 36 | * detected and the stream needs to be aborted. 37 | */ 38 | Optional next() throws UnrecoverableStreamFailureException; 39 | 40 | /** 41 | * Signal the extractor to prepare to extract objects. 42 | * 43 | * @param parentMetrics A parent EtlMetrics object to record all timers and counters into, will be null if 44 | * profiling is not required. 45 | */ 46 | default void open(@Nullable EtlMetrics parentMetrics) { 47 | //no-op 48 | } 49 | 50 | /** 51 | * Signal the extractor that it should free up any resources allocated to the extraction of objects as no more 52 | * requests will be made. 53 | * 54 | * @throws Exception if something goes wrong. 55 | */ 56 | @Override 57 | default void close() throws Exception { 58 | //no-op 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/Loader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import javax.annotation.Nullable; 19 | 20 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 21 | 22 | /** 23 | * Interface for a Loader that loads (writes) objects into final its final destination. 24 | * 25 | * @param Type of object this loader loads. 26 | */ 27 | @FunctionalInterface 28 | public interface Loader extends AutoCloseable { 29 | /** 30 | * Load a single object to to the destination store/service. 31 | * 32 | * @param objectToLoad The object to be loaded. 33 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 34 | * detected and the stream needs to be aborted. 35 | */ 36 | void load(T objectToLoad) throws UnrecoverableStreamFailureException; 37 | 38 | /** 39 | * Signal the loader to prepare to load objects. 40 | * 41 | * @param parentMetrics An EtlMetrics object to attach any child threads created by load() to, will be null if 42 | * profiling is not required. 43 | */ 44 | default void open(@Nullable EtlMetrics parentMetrics) { 45 | //no-op 46 | } 47 | 48 | /** 49 | * Free up any resources allocated for the purposes of loading objects. 50 | * 51 | * @throws Exception if something goes wrong. 52 | */ 53 | @Override 54 | default void close() throws Exception { 55 | //no-op 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/Transformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import javax.annotation.Nullable; 19 | import java.util.List; 20 | 21 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 22 | 23 | /** 24 | * Interface for an object that transforms an object of one type into a stream of objects of another type. This allows 25 | * for an expansion or contraction of the stream where one object can become many objects or no objects. 26 | * 27 | * @param The object type before transformation. 28 | * @param The object type after transformation. 29 | */ 30 | @FunctionalInterface 31 | public interface Transformer extends AutoCloseable { 32 | /** 33 | * Transform a single object. 34 | * 35 | * @param objectToTransform The object to be transformed. 36 | * @return The transformed object. 37 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 38 | * detected and the stream needs to be aborted. 39 | */ 40 | List transform(UpstreamType objectToTransform) throws UnrecoverableStreamFailureException; 41 | 42 | /** 43 | * Signal the transformer to prepare to transform objects. 44 | * 45 | * @param parentMetrics An EtlMetrics object to attach any counters or timers to, will be null if profiling is not 46 | * required. 47 | */ 48 | default void open(@Nullable EtlMetrics parentMetrics) { 49 | //no-op 50 | } 51 | 52 | /** 53 | * Free up any resources allocated for the purposes of transforming objects. 54 | * 55 | * @throws Exception if something goes wrong. 56 | */ 57 | @Override 58 | default void close() throws Exception { 59 | //no-op 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/common/ThrowingFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.common; 17 | 18 | /** 19 | * Simple functional interface for a method that takes a single argument, returns a result and can throw an exception. 20 | * @param Same as Function : Argument class type 21 | * @param Same as Function : Result class type 22 | * @param Checked exception type that the function can throw. 23 | */ 24 | @FunctionalInterface 25 | public interface ThrowingFunction { 26 | /** 27 | * A method that takes a single argument, returns a result and can throw an exception. 28 | * @param t Function argument. 29 | * @return Function result. 30 | * @throws E This function can throw this exception. 31 | */ 32 | R apply(T t) throws E; 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/DefaultLoggingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core; 17 | 18 | import lombok.EqualsAndHashCode; 19 | 20 | import java.util.function.Function; 21 | 22 | /** 23 | * This Logging Strategy is applied if no Logging Strategy was provided by the user. 24 | * @param Type of the object to be logged 25 | */ 26 | @EqualsAndHashCode 27 | public class DefaultLoggingStrategy implements Function { 28 | 29 | private static final String DEFAULT_LOG_MESSAGE = "For more detailed logging for the object that failed, provide a " 30 | + "custom logging strategy to the EtlStage."; 31 | 32 | @Override 33 | public String apply(T t) { 34 | return DEFAULT_LOG_MESSAGE; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/consumer/EtlConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.core.EtlStreamObject; 20 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 21 | 22 | import javax.annotation.Nullable; 23 | 24 | /** 25 | * Common interface for all ETL objects that consume data from the ETL stream. 26 | */ 27 | public interface EtlConsumer extends AutoCloseable { 28 | /** 29 | * Consume a single object. 30 | * 31 | * @param objectToConsume The object to be consumed. 32 | * @throws IllegalStateException If the consumer is in a state that it cannot accept more objects to consume. 33 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 34 | * detected and the stream needs to be aborted. 35 | */ 36 | void consume(EtlStreamObject objectToConsume) throws IllegalStateException, UnrecoverableStreamFailureException; 37 | 38 | /** 39 | * Signal the consumer to prepare receiving work. 40 | * 41 | * @param parentMetrics An EtlMetrics object to store counters and timers. Will be null if profiling is not 42 | * required. 43 | */ 44 | void open(@Nullable EtlMetrics parentMetrics); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/consumer/LogAsErrorEtlConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.EtlProfilingScope; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import lombok.EqualsAndHashCode; 22 | import org.apache.logging.log4j.Logger; 23 | 24 | import javax.annotation.Nonnull; 25 | import java.util.function.Function; 26 | 27 | /** 28 | * Implementation of Consumer that logs the string representation of any objects it consumes to a log4j logger as an 29 | * error. 30 | * 31 | * @param Type of object to be logged 32 | */ 33 | @EqualsAndHashCode 34 | class LogAsErrorEtlConsumer implements EtlConsumer { 35 | private final String name; 36 | private final Logger errorLogger; 37 | private EtlMetrics parentMetrics; 38 | private Class dtoClass; 39 | private Function loggingStrategy; 40 | 41 | /** 42 | * Constructor that creates consumer with logging strategy 43 | * 44 | * @param name A human readable name for the instance of this class that will be used in logging and metrics. 45 | * @param errorLogger Apache log4j logger object to log errors to. 46 | * @param dtoClass Class object to use to create the returned object. 47 | * @param loggingStrategy Function that takes an object from the ETL stream and returns a string representation of that object to be logged. 48 | */ 49 | LogAsErrorEtlConsumer(String name, Logger errorLogger, @Nonnull Class dtoClass, @Nonnull Function loggingStrategy) { 50 | this.name = name; 51 | this.errorLogger = errorLogger; 52 | this.dtoClass = dtoClass; 53 | this.loggingStrategy = loggingStrategy; 54 | } 55 | 56 | /** 57 | * Consumes and logs the string representation of a single object as an error. 58 | * 59 | * @param objectToConsume The object to be consumed. 60 | * @throws IllegalStateException If this consumer is not in a state able to do work. 61 | */ 62 | @Override 63 | public void consume(EtlStreamObject objectToConsume) throws IllegalStateException { 64 | try (EtlProfilingScope ignored = new EtlProfilingScope(parentMetrics, "LogAsErrorConsumer." + name + ".consume")) { 65 | String logMessage = loggingStrategy.apply(objectToConsume.get(dtoClass)); 66 | errorLogger.error("ETL failure for object: " + logMessage); 67 | } catch (RuntimeException e) { 68 | errorLogger.error("ETL failure for object type '" + objectToConsume.getClass().getSimpleName() + 69 | "'. Logging Strategy failed with exception: ", e); 70 | } 71 | } 72 | 73 | /** 74 | * Signal the consumer to prepare to accept work. Does nothing in this implementation. 75 | */ 76 | @Override 77 | public void open(EtlMetrics parentMetrics) { 78 | this.parentMetrics = parentMetrics; 79 | new EtlProfilingScope(parentMetrics, "LogAsErrorConsumer." + name + ".open").close(); 80 | } 81 | 82 | /** 83 | * Signal the consumer to wrap up any buffered work and close and finalize streams. Does nothing in this 84 | * implementation. 85 | * 86 | * @throws Exception If something goes wrong. 87 | */ 88 | @Override 89 | public void close() throws Exception { 90 | new EtlProfilingScope(parentMetrics, "LogAsErrorConsumer." + name + ".close").close(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/consumer/MetricsEmissionEtlConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.EtlProfilingScope; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import lombok.AccessLevel; 22 | import lombok.EqualsAndHashCode; 23 | import lombok.Getter; 24 | 25 | /** 26 | * Implementation of consumer that logs a counter for each record consumed in the format : 27 | * job-name.stage-name.recordsProcessed 28 | * This consumer is automatically inserted into the standard chain of consumers that wrap ETL consumers. 29 | */ 30 | @EqualsAndHashCode 31 | class MetricsEmissionEtlConsumer implements EtlConsumer { 32 | private final String stageName; 33 | 34 | @Getter(AccessLevel.PACKAGE) 35 | private final EtlConsumer downstreamEtlConsumer; 36 | 37 | private EtlMetrics parentMetrics; 38 | 39 | MetricsEmissionEtlConsumer(String stageName, EtlConsumer downstreamEtlConsumer) { 40 | this.stageName = stageName; 41 | this.downstreamEtlConsumer = downstreamEtlConsumer; 42 | } 43 | 44 | @Override 45 | public void consume(EtlStreamObject objectToConsume) throws IllegalStateException { 46 | try (EtlProfilingScope scope = new EtlProfilingScope(parentMetrics, "MetricsEmissionConsumer." + stageName + ".consume")) { 47 | scope.addCounter(stageName + ".recordsProcessed", 1); 48 | downstreamEtlConsumer.consume(objectToConsume); 49 | } 50 | } 51 | 52 | @Override 53 | public void open(EtlMetrics parentMetrics) { 54 | try (EtlProfilingScope scope = new EtlProfilingScope(parentMetrics, "MetricsEmissionConsumer." + 55 | stageName + 56 | ".open")) { 57 | scope.addCounter(stageName + ".recordsProcessed", 0); 58 | this.parentMetrics = parentMetrics; 59 | downstreamEtlConsumer.open(parentMetrics); 60 | } 61 | } 62 | 63 | @Override 64 | public void close() throws Exception { 65 | downstreamEtlConsumer.close(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/executor/EtlExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.exception.GenericEtlException; 20 | 21 | import java.util.concurrent.RejectedExecutionException; 22 | 23 | /** 24 | * An abstracted interface for an executor that can schedule, queue and perform work. 25 | */ 26 | public interface EtlExecutor { 27 | /** 28 | * Complete any remaining work scheduled by the executor and then shut it down. 29 | * 30 | * @throws GenericEtlException If something goes wrong. 31 | */ 32 | void shutdown() throws GenericEtlException; 33 | 34 | /** 35 | * Queries whether the executor has been shutdown. 36 | * 37 | * @return 'true' if the executor has been shutdown, and 'false' if it has not. 38 | */ 39 | boolean isShutdown(); 40 | 41 | /** 42 | * Schedules a task to be worked on by the Executor. Depending on the implementation this work may be scheduled, 43 | * worked in parallel with other tasks or block until other work has been completed first. No guarantees are 44 | * made about when the work will get done. 45 | * 46 | * @param task a runnable wrapping the task to be performed in the future. 47 | * @param parentMetrics A parent EtlMetrics object to attach the runnables to. 48 | * @throws RejectedExecutionException If the task cannot be submitted to the Executor. 49 | */ 50 | void submit(Runnable task, EtlMetrics parentMetrics) throws RejectedExecutionException; 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/executor/EtlExecutorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | import java.util.concurrent.ArrayBlockingQueue; 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.Executors; 21 | import java.util.concurrent.RejectedExecutionException; 22 | import java.util.concurrent.ThreadPoolExecutor; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | /** 26 | * An injectable factory class for building various types of useful EtlExecutor implementations. This is the only way 27 | * you should be constructing EtlExecutor objects of any kind outside of this package. 28 | */ 29 | public class EtlExecutorFactory { 30 | /** 31 | * This multi-threaded EtlExecutor uses a fixed-size work queue that will block on new requests once the queue is 32 | * full until it drains enough to add the new task. 33 | * @param numberOfWorkers Number of threads to run tasks simultaneously. 34 | * @param queueSize The maximum size of the work-queue. Submit will block once this hits its size limit. 35 | * @return A fully constructed EtlExecutor. 36 | */ 37 | public EtlExecutor newBlockingFixedThreadsEtlExecutor(int numberOfWorkers, int queueSize) { 38 | ExecutorService executorService = new ThreadPoolExecutor(numberOfWorkers, numberOfWorkers, Long.MAX_VALUE, 39 | TimeUnit.NANOSECONDS, new ArrayBlockingQueue<>(queueSize), (runnable, threadPoolExecutor) -> { 40 | if (threadPoolExecutor.isShutdown()) { 41 | throw new RejectedExecutionException("ExecutorService was shutdown"); 42 | } 43 | try { 44 | threadPoolExecutor.getQueue().put(runnable); 45 | } catch (InterruptedException ignored) { 46 | throw new RejectedExecutionException("Thread was interrupted trying to queue new work"); 47 | } 48 | }); 49 | 50 | return new ExecutorServiceEtlExecutor(executorService); 51 | } 52 | 53 | /** 54 | * This multi-threaded EtlExecutor uses an unbound queue that will not block on new requests. 55 | * @param numberOfWorkers Number of threads to run tasks simultaneously. 56 | * @return A fully constructed EtlExecutor. 57 | */ 58 | public EtlExecutor newUnboundFixedThreadsEtlExecutorFactory(int numberOfWorkers) { 59 | return new ExecutorServiceEtlExecutor(Executors.newFixedThreadPool(numberOfWorkers)); 60 | } 61 | 62 | /** 63 | * This is a single-threaded EtlExecutor used when you don't want any kind of parallelism, but conceptually will 64 | * behave like other EtlExecutors. Executions will block until completed by the invoking thread. This should be 65 | * used with caution because it does not protect downstream consumers against unwanted parallelism. 66 | * @return A fully constructed EtlExecutor. 67 | */ 68 | public EtlExecutor newImmediateExecutionEtlExecutor() { 69 | return new ImmediateExecutionEtlExecutor(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/executor/ExecutorServiceEtlExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.EtlProfilingScope; 20 | import com.amazon.pocketEtl.exception.GenericEtlException; 21 | import lombok.AccessLevel; 22 | import lombok.EqualsAndHashCode; 23 | import lombok.Getter; 24 | 25 | import java.util.concurrent.ExecutorService; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | /** 29 | * An EtlExecutor implementation that wraps an ExecutorService and can be used to parallelize work from a queue. This 30 | * object should not be constructed directly, instead use EtlExecutorFactory. 31 | */ 32 | @EqualsAndHashCode(exclude = {"executorService"}) 33 | class ExecutorServiceEtlExecutor implements EtlExecutor { 34 | @Getter(AccessLevel.PACKAGE) 35 | private final ExecutorService executorService; 36 | 37 | ExecutorServiceEtlExecutor(ExecutorService executorService) { 38 | this.executorService = executorService; 39 | } 40 | 41 | /** 42 | * Drain the work-queue and destroy the thread resources used by this object. This request will block until 43 | * the threads have finished working. 44 | * 45 | * @throws GenericEtlException If this thread was interrupted whilst waiting for the shutdown. 46 | */ 47 | @Override 48 | public void shutdown() throws GenericEtlException { 49 | executorService.shutdown(); 50 | 51 | try { 52 | executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); 53 | } catch (InterruptedException ignored) { 54 | } 55 | 56 | if (!executorService.isTerminated()) { 57 | executorService.shutdownNow(); 58 | 59 | try { 60 | executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); 61 | } catch (InterruptedException ignored) { 62 | } 63 | 64 | if (!executorService.isTerminated()) { 65 | throw new GenericEtlException("Timed out waiting for forced shutdown of executor service"); 66 | } 67 | } 68 | } 69 | 70 | /** 71 | * Queries whether the executor has been shutdown. 72 | * 73 | * @return 'true' if the executor has been shutdown, and 'false' if it has not. 74 | */ 75 | @Override 76 | public boolean isShutdown() { 77 | return executorService.isShutdown(); 78 | } 79 | 80 | /** 81 | * Submits a task to the work queue of the Executor. The task will be worked on at some point in the future by 82 | * one of the threads being managed by the executor unless the executor has been shutdown first. No guarantees are 83 | * made about when the work will get done. 84 | * 85 | * @param task a runnable wrapping the task to be performed in the future. 86 | * @param parentMetrics A parent EtlMetrics object to attach the runnables to. 87 | */ 88 | @Override 89 | public void submit(Runnable task, EtlMetrics parentMetrics) { 90 | executorService.submit(() -> { 91 | try (EtlProfilingScope ignored = new EtlProfilingScope(parentMetrics, "ExecutorServiceEtlExecutor.submit")) { 92 | task.run(); 93 | } 94 | }); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/executor/ImmediateExecutionEtlExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | 19 | import com.amazon.pocketEtl.EtlMetrics; 20 | import com.amazon.pocketEtl.EtlProfilingScope; 21 | import com.amazon.pocketEtl.exception.GenericEtlException; 22 | import lombok.EqualsAndHashCode; 23 | 24 | import java.util.concurrent.RejectedExecutionException; 25 | import java.util.concurrent.atomic.AtomicBoolean; 26 | 27 | /** 28 | * Single-threaded implementation of EtlExecutor. Runnables will run in the same thread that submits them. 29 | */ 30 | @EqualsAndHashCode(exclude = {"isShutdown"}) 31 | class ImmediateExecutionEtlExecutor implements EtlExecutor { 32 | private final AtomicBoolean isShutdown = new AtomicBoolean(false); 33 | 34 | /** 35 | * Shuts down the EtlExecutor. For this implementation it's a no-op. 36 | * @throws GenericEtlException If something goes wrong. 37 | */ 38 | @Override 39 | public void shutdown() throws GenericEtlException { 40 | isShutdown.set(true); 41 | } 42 | 43 | /** 44 | * Returns the shutdown status of the EtlExecutor. 45 | * @return true if the EtlExecutor has been shutdown; false if it has not. 46 | */ 47 | @Override 48 | public boolean isShutdown() { 49 | return isShutdown.get(); 50 | } 51 | 52 | /** 53 | * Submits a single task to the Executor. This will be executed immediately in the same thread that is calling 54 | * this method and will not return until the runnable has completed. Any exception thrown by the runnable will 55 | * be swallowed just like a thread-pool would. 56 | * @param task a runnable wrapping the task to be performed in the future. 57 | * @param parentMetrics A parent EtlMetrics object to attach the runnables to. 58 | */ 59 | @Override 60 | public void submit(Runnable task, EtlMetrics parentMetrics) { 61 | if (isShutdown.get()) { 62 | throw new RejectedExecutionException("Executor has been shutdown and cannot accept more work"); 63 | } 64 | 65 | try (EtlProfilingScope ignored = new EtlProfilingScope(parentMetrics, "SingleThreadedEtlExecutor.submit")) { 66 | task.run(); 67 | } catch (RuntimeException ignored) { 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/producer/EtlProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.producer; 17 | 18 | import javax.annotation.Nullable; 19 | 20 | import com.amazon.pocketEtl.EtlMetrics; 21 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 22 | 23 | /** 24 | * Producer interface for ETL system. A producer will, once started, continue to produce work until the supply of 25 | * work has been exhausted. Typically a producer will send work it produces to a downstream consumer of some kind. 26 | */ 27 | public interface EtlProducer extends AutoCloseable { 28 | /** 29 | * Produce objects until the source of objects has been exhausted. 30 | * 31 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 32 | * detected and the stream needs to be aborted. 33 | * @throws IllegalStateException If the producer is not in a state capable of producing new work. 34 | */ 35 | void produce() throws UnrecoverableStreamFailureException, IllegalStateException; 36 | 37 | /** 38 | * Signal the producer to complete its work and free any resources allocated for the production of work. 39 | * 40 | * @throws Exception If something goes wrong. 41 | */ 42 | @Override 43 | void close() throws Exception; 44 | 45 | /** 46 | * Signal the producer that it should prepare to produce work. 47 | * @param parentMetrics An EtlMetrics object to add counters and timers to, will be null if profiling is not 48 | * required. 49 | */ 50 | void open(@Nullable EtlMetrics parentMetrics); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/core/producer/EtlProducerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.producer; 17 | 18 | import com.amazon.pocketEtl.Extractor; 19 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 20 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 21 | import lombok.RequiredArgsConstructor; 22 | 23 | import java.util.Collection; 24 | 25 | /** 26 | * An injectable factory class for building various types of useful Producer implementations. This is the only 27 | * way you should be constructing Producer objects of any kind outside of this package. Producers of various kinds 28 | * should be combined into a single Producer using combineProducers() which will then run all the producers it is 29 | * constructed with together. 30 | */ 31 | @RequiredArgsConstructor 32 | public class EtlProducerFactory { 33 | private final EtlExecutorFactory etlExecutorFactory; 34 | 35 | /** 36 | * Constructs a new producer based on an Extractor. 37 | * @param name The name of this producer used in logging and reporting. 38 | * @param extractor The extractor object this producer is based on. 39 | * @param downstreamEtlConsumer The consumer to send objects this producer extracts to. 40 | * @param The type of object being extracted. 41 | * @return A fully constructed producer. 42 | */ 43 | public EtlProducer newExtractorProducer(String name, Extractor extractor, EtlConsumer downstreamEtlConsumer) { 44 | return new ExtractorEtlProducer<>(name, downstreamEtlConsumer, extractor); 45 | } 46 | 47 | /** 48 | * Combines multiple producers into a single producer object that behaves like a single producer but drives all 49 | * the producers it was constructed with. 50 | * @param name The name of this producer used in logging and reporting. 51 | * @param etlProducers A collection of producers to combine into a single producer. 52 | * @param numberOfParallelWorkers The degree of parallelism to drive the producers with. 53 | * @return A fully constructed producer. 54 | */ 55 | public EtlProducer combineProducers(String name, Collection etlProducers, int numberOfParallelWorkers) { 56 | return new ExecutorEtlProducer(name, etlProducers, etlExecutorFactory.newUnboundFixedThreadsEtlExecutorFactory(numberOfParallelWorkers)); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/exception/DependencyException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.exception; 17 | 18 | /** 19 | * Exception thrown when a dependency is missing. For example, null Connection is returned by database dataSource. 20 | */ 21 | public class DependencyException extends RuntimeException { 22 | /** 23 | * Standard constructor. 24 | * @param s Exception message giving more details of the problem. 25 | */ 26 | public DependencyException(String s) { 27 | super(s); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/exception/GenericEtlException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.exception; 17 | 18 | /** 19 | * Exception thrown when there is a problem in the underlying service. 20 | */ 21 | public class GenericEtlException extends Exception { 22 | /** 23 | * Standard constructor. 24 | * @param s Exception message giving more details of the problem. 25 | */ 26 | public GenericEtlException(String s) { 27 | super(s); 28 | } 29 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/exception/UnrecoverableStreamFailureException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.exception; 17 | 18 | /** 19 | * Exception that can be thrown by any step in a job when an unrecoverable problem has occurred and the stream needs to 20 | * be aborted at the first opportunity. An example would be losing access to a dependent resource such as a database 21 | * or filesystem or a problem that is likely to effect an entire class or batch of objects rather than a single 22 | * object such as a batched flush failing on a loader. 23 | * 24 | * If any step in the ETL stream throws this exception then it will immediately propagate through the stream shutting 25 | * down every upstream step and eventually failing the entire job. 26 | */ 27 | public class UnrecoverableStreamFailureException extends RuntimeException { 28 | /** 29 | * Standard constructor. 30 | * @param s Exception message giving more details of the problem. 31 | */ 32 | public UnrecoverableStreamFailureException(String s) { 33 | super(s); 34 | } 35 | 36 | /** 37 | * Standard constructor. 38 | * @param cause Exception that caused this failure. 39 | */ 40 | public UnrecoverableStreamFailureException(Throwable cause) { 41 | super(cause); 42 | } 43 | 44 | /** 45 | * Standard constructor. 46 | * @param message Exception message giving more details of the problem. 47 | * @param cause Exception that caused this failure. 48 | */ 49 | public UnrecoverableStreamFailureException(String message, Throwable cause) { 50 | super(message, cause); 51 | } 52 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/CsvInputStreamMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.fasterxml.jackson.databind.ObjectReader; 19 | import com.fasterxml.jackson.dataformat.csv.CsvMapper; 20 | import com.fasterxml.jackson.dataformat.csv.CsvParser; 21 | import com.fasterxml.jackson.dataformat.csv.CsvSchema; 22 | 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.util.Iterator; 26 | 27 | /** 28 | * Implementation of InputStreamMapper that takes an InputStream with CSV formatted data in it and creates an 29 | * iterator around that data. 30 | * 31 | * Powered by Jackson's CsvMapper class, so for precise behavioral information you can refer to their documentation. 32 | * @param The type of object the CSV data is being mapped to. 33 | */ 34 | @SuppressWarnings("WeakerAccess") 35 | public class CsvInputStreamMapper implements InputStreamMapper { 36 | private static final CsvMapper mapper = new CsvMapper(); 37 | 38 | private final Class objectClass; 39 | private Character columnSeparator = null; 40 | 41 | /** 42 | * Convert an InputStream containing CSV data to an iterator. Used by InputStreamExtractor. 43 | *

44 | * Example usage: 45 | * CsvInputStreamMapper.fromCSV(My.class) 46 | * 47 | * @param objectClassToMapTo Class definition to map the CSV data into 48 | * @param Type of object being iterated over. 49 | * @return A function that converts an inputStream to an iterator. 50 | */ 51 | public static CsvInputStreamMapper of(Class objectClassToMapTo) { 52 | return new CsvInputStreamMapper<>(objectClassToMapTo); 53 | } 54 | 55 | /** 56 | * Change the character that is used to delimit the columns in the data on the InputStream. By default this value 57 | * is a comma. 58 | * @param columnSeparator The character to treat as a column delimiter. 59 | * @return A copy of this object with this property changed. 60 | */ 61 | public CsvInputStreamMapper withColumnSeparator(Character columnSeparator) { 62 | this.columnSeparator = columnSeparator; 63 | return this; 64 | } 65 | 66 | /** 67 | * Converts an inputStream into an iterator of objects using Jackson CSV mapper. 68 | * @param inputStream inputStream in CSV format. 69 | * @return An iterator based on the inputStream. 70 | */ 71 | @Override 72 | public Iterator apply(InputStream inputStream) { 73 | try { 74 | CsvSchema csvSchema = mapper.schemaFor(objectClass); 75 | 76 | if (columnSeparator != null) { 77 | csvSchema = csvSchema.withColumnSeparator(columnSeparator); 78 | } 79 | 80 | ObjectReader reader = mapper.readerFor(objectClass).withFeatures(CsvParser.Feature.FAIL_ON_MISSING_COLUMNS) 81 | .with(csvSchema); 82 | 83 | return reader.readValues(inputStream); 84 | } catch (IOException e) { 85 | throw new RuntimeException(e); 86 | } 87 | } 88 | 89 | private CsvInputStreamMapper( 90 | Class objectClass 91 | ) { 92 | this.objectClass = objectClass; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/InputStreamExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import java.io.InputStream; 19 | import java.util.Iterator; 20 | import java.util.Optional; 21 | import java.util.function.Supplier; 22 | 23 | import com.amazon.pocketEtl.EtlMetrics; 24 | import com.amazon.pocketEtl.Extractor; 25 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 26 | 27 | import lombok.AccessLevel; 28 | import lombok.RequiredArgsConstructor; 29 | 30 | /** 31 | * Extractor implementation that maps an input stream into objects to be extracted. 32 | * 33 | * Example usage: 34 | * InputStreamExtractor.of(S3BufferedInputStream.of("myBucket", "myFile"), CsvInputStreamMapper.of(My.class)); 35 | * 36 | * This example will create an extractor that will read a CSV file called 'myFile' stored in the S3 bucket 'myBucket' 37 | * and map each row to a newly constructed My.class object. 38 | * 39 | * @param Type of object to be extracted from the input stream. 40 | */ 41 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 42 | public class InputStreamExtractor implements Extractor { 43 | private final Supplier inputStreamSupplier; 44 | private final InputStreamMapper inputStreamMapperFunction; 45 | 46 | private InputStream inputStream = null; 47 | private Iterator mappingIterator = null; 48 | 49 | /** 50 | * Static constructor method to build a functioning extractor. 51 | * @param inputStreamProvider A function that supplies the inputStream to be extracted from. 52 | * @param inputStreamMapper A function that can map an open inputStream into an iterator of the objects being 53 | * extracted. 54 | * @param The type of object being extracted/mapped-to. 55 | * @return A functioning extractor. 56 | */ 57 | public static Extractor of(Supplier inputStreamProvider, 58 | InputStreamMapper inputStreamMapper) { 59 | return new InputStreamExtractor<>(inputStreamProvider, inputStreamMapper); 60 | } 61 | 62 | /** 63 | * This implementation of open will construct an inputStream and then create an iterator from it. 64 | * @param parentMetrics A parent EtlMetrics object to record all timers and counters into, will be null if 65 | */ 66 | @Override 67 | public void open(EtlMetrics parentMetrics) { 68 | inputStream = inputStreamSupplier.get(); 69 | mappingIterator = inputStreamMapperFunction.apply(inputStream); 70 | } 71 | 72 | /** 73 | * Attempts to extract the next object from the inputStream. 74 | * @return A newly extracted object or an empty optional if the end of the stream has been reached. 75 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 76 | * detected and the stream needs to be aborted. 77 | */ 78 | @Override 79 | public Optional next() throws UnrecoverableStreamFailureException { 80 | if (inputStream == null || mappingIterator == null) { 81 | throw new IllegalStateException("Attempt to call next() on an uninitialized stream"); 82 | } 83 | 84 | try { 85 | if (mappingIterator.hasNext()) { 86 | T nextObject = mappingIterator.next(); 87 | return Optional.of(nextObject); 88 | } 89 | else { 90 | return Optional.empty(); 91 | } 92 | } catch (RuntimeException e) { 93 | throw new UnrecoverableStreamFailureException(e); 94 | } 95 | } 96 | 97 | /** 98 | * Signals the extractor that it should free up any resources used to extract new objects. This will close the 99 | * inputStream that was being wrapped by this extractor. 100 | * @throws Exception If something goes wrong. 101 | */ 102 | @Override 103 | public void close() throws Exception { 104 | if (inputStream != null) { 105 | inputStream.close(); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/InputStreamMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import java.io.InputStream; 19 | import java.util.Iterator; 20 | import java.util.function.Function; 21 | 22 | /** 23 | * Functional interface that maps an InputStream to an Iterator of a specific type. Used by InputStreamExtractor. 24 | * @param Type of object being mapped to. 25 | */ 26 | @SuppressWarnings("WeakerAccess") 27 | @FunctionalInterface 28 | public interface InputStreamMapper extends Function> { 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/IterableExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 21 | 22 | import lombok.AccessLevel; 23 | import lombok.RequiredArgsConstructor; 24 | 25 | import javax.annotation.Nullable; 26 | import java.util.Iterator; 27 | import java.util.Optional; 28 | 29 | /** 30 | * An Extractor implementation that wraps any Java iterable object. The iterator will be created just once when the ETL 31 | * job is first run. 32 | */ 33 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 34 | public class IterableExtractor implements Extractor { 35 | private final Iterable iterable; 36 | private Iterator iterator = null; 37 | 38 | /** 39 | * Simple static builder that constructs an extractor around a Java Iterable object. 40 | * @param iterable An iterable object. 41 | * @param Type of object the iterator iterates on. 42 | * @return An Extractor object that can be used in Pocket ETL jobs. 43 | */ 44 | public static Extractor of(Iterable iterable) { 45 | return new IterableExtractor<>(iterable); 46 | } 47 | 48 | /** 49 | * Get the next object from the wrapped iterable. If there are any problems, throw an 50 | * UnrecoverableStreamFailureException to force the ETL runner to pay attention. 51 | * @return An optional extracted object. Empty if there are no objects left to extract. 52 | * @throws UnrecoverableStreamFailureException An unrecoverable problem that affects the entire stream has been 53 | * detected and the stream needs to be aborted. 54 | */ 55 | @Override 56 | public Optional next() throws UnrecoverableStreamFailureException { 57 | if (iterator == null) { 58 | throw new IllegalStateException("next() called on an unopened extractor"); 59 | } 60 | 61 | try { 62 | return iterator.hasNext() ? Optional.of(iterator.next()) : Optional.empty(); 63 | } catch (RuntimeException e) { 64 | throw new UnrecoverableStreamFailureException(e); 65 | } 66 | } 67 | 68 | /** 69 | * Create the iterator from the wrapped iterable object and prepare to start iterating on it. 70 | * @param parentMetrics A parent EtlMetrics object to record all timers and counters into, will be null if 71 | * profiling is not required. 72 | */ 73 | @Override 74 | public void open(@Nullable EtlMetrics parentMetrics) { 75 | iterator = iterable.iterator(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/IteratorExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import java.util.Iterator; 19 | import java.util.Optional; 20 | 21 | import com.amazon.pocketEtl.Extractor; 22 | 23 | import lombok.AccessLevel; 24 | import lombok.RequiredArgsConstructor; 25 | 26 | /** 27 | * Simple static builder that constructs an extractor around a simple Java Iterator object. 28 | */ 29 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 30 | public class IteratorExtractor implements Extractor { 31 | private final Iterator iterator; 32 | 33 | /** 34 | * Simple static builder that constructs an extractor around a simple Java Iterator object. 35 | * @param iterator A standard iterator. 36 | * @param Type of object the iterator iterates on. 37 | * @return An Extractor object that can be used in Pocket ETL jobs. 38 | */ 39 | public static Extractor of(Iterator iterator) { 40 | return new IteratorExtractor<>(iterator); 41 | } 42 | 43 | @Override 44 | public Optional next() { 45 | return iterator.hasNext() ? Optional.of(iterator.next()) : Optional.empty(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/JSONStringMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.fasterxml.jackson.databind.JsonMappingException; 19 | import com.fasterxml.jackson.databind.MapperFeature; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.fasterxml.jackson.databind.ObjectReader; 22 | import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; 23 | import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; 24 | import com.fasterxml.jackson.datatype.joda.JodaModule; 25 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 26 | 27 | import java.io.IOException; 28 | import java.util.function.Function; 29 | 30 | /** 31 | * A handy mapper function for mapping any json having one json object to a DTO of given type. Powered by Jackson, so 32 | * you can use Jackson annotations on your DTO class. 33 | * 34 | * Usage: 35 | * JSONStringMapper.of(MyDTO.class); 36 | * 37 | * Example valid JSON input: 38 | * { 39 | * "keyOne": "valueOne", 40 | * "keyTwo": "valueTwo" 41 | * } 42 | * 43 | * NOTE: Currently this mapper is able to map string dates in json to joda DateTime which are in ISO format. 44 | * For example, "2017-08-15T12:00:00Z" 45 | * This is an open bug with jackson API. This needs to be handled in future by adding custom joda datetime 46 | * serializable or wait for jackson API to fix this. 47 | */ 48 | public class JSONStringMapper implements Function { 49 | private static final ObjectMapper objectMapper = new ObjectMapper() 50 | // Mapping java bean case conventions can be problematic when the second character is capitalized 51 | // (eg: getAString()). We are solving this problem by making case not matter. 52 | .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true) 53 | .registerModules(new JodaModule(), new Jdk8Module(), new JavaTimeModule()); 54 | 55 | private final ObjectReader objectReader; 56 | 57 | public static JSONStringMapper of(Class mapToClass) { 58 | return new JSONStringMapper<>(mapToClass); 59 | } 60 | 61 | private JSONStringMapper(Class mapToClass) { 62 | objectReader = objectMapper.readerFor(mapToClass); 63 | } 64 | 65 | /** 66 | * Parse the json string and creates an object of dtoClass with field values populated from parsed json field values. 67 | * @param json json string to be mapped. 68 | * @return object of dtoClass type having values populated from json string. 69 | */ 70 | @Override 71 | public T apply(String json) { 72 | if (json == null) { 73 | return null; 74 | } 75 | 76 | try { 77 | return objectReader.readValue(json); 78 | } catch (UnrecognizedPropertyException e) { 79 | throw new RuntimeException("Unable to find property in mapToClass. " + 80 | "Check for typos or make sure dtoClass has all the fields that are in json: ", e); 81 | } catch (JsonMappingException e) { 82 | throw new RuntimeException("Make sure the json is valid and contains only one object: ", e); 83 | } catch (IOException e) { 84 | throw new RuntimeException("Unable to parse JSON string: ", e); 85 | } 86 | } 87 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/WrappedExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import static lombok.AccessLevel.PROTECTED; 19 | 20 | import java.util.Optional; 21 | 22 | import javax.annotation.Nullable; 23 | 24 | import com.amazon.pocketEtl.EtlMetrics; 25 | import com.amazon.pocketEtl.Extractor; 26 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 27 | 28 | import lombok.RequiredArgsConstructor; 29 | 30 | /** 31 | * Allows a class to masquerade as an Extractor by extending this class and providing an implementation of a method 32 | * to get the real Extractor it is masquerading as. Used by classes that build complex Extractor objects but don't have 33 | * any real implementation themselves such as S3BufferedExtractor. 34 | * @param The type of object being extracted. 35 | */ 36 | @RequiredArgsConstructor(access = PROTECTED) 37 | public abstract class WrappedExtractor implements Extractor { 38 | protected abstract Extractor getWrappedExtractor(); 39 | 40 | @Override 41 | public Optional next() throws UnrecoverableStreamFailureException { 42 | return getWrappedExtractor().next(); 43 | } 44 | 45 | @Override 46 | public void open(@Nullable EtlMetrics parentMetrics) { 47 | getWrappedExtractor().open(parentMetrics); 48 | } 49 | 50 | @Override 51 | public void close() throws Exception { 52 | getWrappedExtractor().close(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/extractor/WrappedInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import lombok.AccessLevel; 19 | import lombok.NoArgsConstructor; 20 | 21 | import javax.annotation.Nonnull; 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | 25 | /** 26 | * Allows a class to masquerade as an InputStream by extending this class and providing an implementation of a method 27 | * to get the real InputStream it is masquerading as. 28 | * 29 | * Mark and reset are not intentionally not supported by this wrapper due to: 30 | * a) Not being needed. 31 | * b) Requiring remapping of IOException that might be thrown by getWrappedInputStream(). The reason 32 | * getWrappedInputStream() might throw an IOException is to support implementation of eagerly buffered InputStream 33 | * implementations such as the S3BufferedInputStream. 34 | */ 35 | @NoArgsConstructor(access = AccessLevel.PROTECTED) 36 | public abstract class WrappedInputStream extends InputStream { 37 | protected abstract InputStream getWrappedInputStream() throws IOException; 38 | 39 | @Override 40 | public int read(@Nonnull byte[] b) throws IOException { 41 | return getWrappedInputStream().read(b); 42 | } 43 | 44 | @Override 45 | public int read(@Nonnull byte[] b, int off, int len) throws IOException { 46 | return getWrappedInputStream().read(b, off, len); 47 | } 48 | 49 | @Override 50 | public int read() throws IOException { 51 | return getWrappedInputStream().read(); 52 | } 53 | 54 | @Override 55 | public long skip(long n) throws IOException { 56 | return getWrappedInputStream().skip(n); 57 | } 58 | 59 | @Override 60 | public int available() throws IOException { 61 | return getWrappedInputStream().available(); 62 | } 63 | 64 | @Override 65 | public void close() throws IOException { 66 | getWrappedInputStream().close(); 67 | } 68 | 69 | @Override 70 | public boolean markSupported() { 71 | return false; 72 | } 73 | 74 | @Override 75 | public synchronized void mark(int readlimit) { 76 | throw new UnsupportedOperationException(); 77 | } 78 | 79 | @Override 80 | public synchronized void reset() { 81 | throw new UnsupportedOperationException(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/integration/db/SqlStringHelpers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.integration.db; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.concurrent.atomic.AtomicReference; 21 | 22 | /** 23 | * An Helper class which exposes static methods for substituting values in SQL string which 24 | * normally can't be done with the APIs that are currently being used in the package. 25 | */ 26 | public class SqlStringHelpers { 27 | 28 | /** 29 | * Creates a comma separated string-list, with each element wrapped in single quotes. 30 | * 31 | * @param stringList - A list of Strings (e.g. ["a", "b", "c"] ) 32 | * @return A comma separated string-list (e.g. "'a', 'b', 'c'" ) 33 | */ 34 | public static String listToSQLString(List stringList) { 35 | if (stringList == null) { return ""; } 36 | 37 | StringBuilder stringBuilder = new StringBuilder(); 38 | boolean isFirst = true; 39 | 40 | for (String value : stringList) { 41 | if (!isFirst) { 42 | stringBuilder.append(", "); 43 | } 44 | 45 | stringBuilder.append('\'').append(value).append('\''); 46 | isFirst = false; 47 | } 48 | 49 | return stringBuilder.toString(); 50 | } 51 | 52 | /** 53 | * Replaces key with corresponding value in the SQL string. 54 | * Keys are enclosed by ${} in SQL string. For e.g: 55 | * firstName is the key in "SELECT * from User where firstName=${firstName}". 56 | * 57 | * @param originalSql SQL string having keys that needs to be replaced with the corresponding value. 58 | * @param keyValueSubstitutions Key-Value pairs. 59 | * @return An SQL string with substitutions filled in. 60 | */ 61 | public static String substituteSQL(String originalSql, Map keyValueSubstitutions) { 62 | if (keyValueSubstitutions == null) { return originalSql; } 63 | 64 | AtomicReference result = new AtomicReference<>(originalSql); 65 | 66 | keyValueSubstitutions.forEach((key, value) -> { 67 | String substitutionKey = String.format("${%s}", key); 68 | result.set(result.get().replace(substitutionKey, value)); 69 | }); 70 | 71 | return result.get(); 72 | } 73 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/integration/db/jdbi/EtlBeanMapperFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.integration.db.jdbi; 17 | 18 | import lombok.AllArgsConstructor; 19 | import org.skife.jdbi.v2.ResultSetMapperFactory; 20 | import org.skife.jdbi.v2.StatementContext; 21 | import org.skife.jdbi.v2.tweak.ResultSetMapper; 22 | 23 | import java.util.Map; 24 | import java.util.function.BiConsumer; 25 | 26 | import static lombok.AccessLevel.PACKAGE; 27 | 28 | /** 29 | * ResultSetMapperFactory implementation that will accept any class to be mapped and construct an EtlBeanMapper on 30 | * demand. 31 | */ 32 | @AllArgsConstructor(access = PACKAGE) 33 | class EtlBeanMapperFactory implements ResultSetMapperFactory { 34 | private BiConsumer> secondaryMapper; 35 | 36 | @Override 37 | public boolean accepts(Class type, StatementContext ctx) { 38 | return true; 39 | } 40 | 41 | @Override 42 | @SuppressWarnings("unchecked") 43 | public ResultSetMapper mapperFor(Class type, StatementContext ctx) { 44 | return new EtlBeanMapper<>(type, secondaryMapper); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/integration/db/jdbi/EtlJdbi.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.integration.db.jdbi; 17 | 18 | import org.skife.jdbi.v2.DBI; 19 | import org.skife.jdbi.v2.HashPrefixStatementRewriter; 20 | 21 | import javax.annotation.Nullable; 22 | import javax.sql.DataSource; 23 | import java.util.Map; 24 | import java.util.function.BiConsumer; 25 | 26 | /** 27 | * JDBI integration factory. Constructs a DBI that has custom handlers for things we care about: 28 | * - Joda DateTime marshalling from ResultSet. 29 | * - String list SQL parameter handling (Postgres/Redshift only) 30 | */ 31 | public class EtlJdbi { 32 | /** 33 | * Construct a new DBI wrapper for a JDBC datasource using an optional secondary mapper. The secondary mapper comes 34 | * into play when the default mapper is unable to directly map a value that the SQL query returned to a property in 35 | * the bean class you have specified. In this case you give it a lambda to call that will insert the value correctly 36 | * into the bean object. 37 | * @param dataSource JDBC datasource to be wrapped. 38 | * @param secondaryMapper A lambda that is invoked when a data element can't be directly mapped to the bean is 39 | * extracted. The lambda takes as its arguments the bean that is being extracted to and 40 | * a Map.Entry that represents the (key, value) pair of the data element that could not be 41 | * directly mapped to the bean. If set to null, unrecognized properties will be ignored. 42 | * @return A fully constructed DBI object. 43 | */ 44 | public static DBI newDBI(DataSource dataSource, @Nullable BiConsumer> secondaryMapper) { 45 | DBI dbi = new DBI(dataSource); 46 | dbi.setStatementRewriter(new HashPrefixStatementRewriter()); 47 | dbi.registerMapper(new EtlBeanMapperFactory(secondaryMapper)); 48 | dbi.registerArgumentFactory(new PostgresStringArrayArgumentFactory()); 49 | 50 | return dbi; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/integration/db/jdbi/PostgresStringArrayArgumentFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.integration.db.jdbi; 17 | 18 | import org.skife.jdbi.v2.StatementContext; 19 | import org.skife.jdbi.v2.tweak.Argument; 20 | import org.skife.jdbi.v2.tweak.ArgumentFactory; 21 | 22 | import java.sql.Array; 23 | import java.util.List; 24 | 25 | /** 26 | * JDBI ArgumentFactory implementation that takes a string list argument and uses JDBC createArrayOf to pass it in as a 27 | * preparedStatement parameter. 28 | * CAUTION: This particular implementation is Postgres/Redshift specific and is not believed to be portable to other 29 | * databases. 30 | */ 31 | class PostgresStringArrayArgumentFactory implements ArgumentFactory> { 32 | @Override 33 | public boolean accepts(Class expectedType, Object value, StatementContext ctx) { 34 | // Check if the object is a list 35 | if (!List.class.isAssignableFrom(value.getClass())) { 36 | return false; 37 | } 38 | 39 | List untypedList = (List) value; 40 | 41 | // Check that each object in the list can be assigned into a String 42 | for (Object obj : untypedList) { 43 | if (!String.class.isAssignableFrom(obj.getClass())) { 44 | return false; 45 | } 46 | } 47 | 48 | return true; 49 | } 50 | 51 | @Override 52 | public Argument build(Class expectedType, 53 | final List value, 54 | StatementContext ctx) { 55 | return (position, statement, ctx1) -> { 56 | // in postgres no need to (and in fact cannot) free arrays 57 | Array ary = ctx1.getConnection() 58 | .createArrayOf("text", value.toArray()); 59 | statement.setArray(position, ary); 60 | }; 61 | } 62 | } 63 | 64 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/loader/S3PartFileKeyGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.loader; 17 | 18 | import javax.annotation.Nonnull; 19 | 20 | /** 21 | * Functional interface that describes a method that generates an s3Key (analogous to canonical filename) based on 22 | * two sequence numbers: thread identifier and file part number. Used by S3 based loaders such as S3FastLoader. 23 | * 24 | * Example: 25 | * (threadNum, partNum) -> String.format("etl-output/%02d/data-%d.csv") 26 | * 27 | * Will typically produce keys that look like : 28 | * etl-output/01/data-1.csv 29 | * etl-output/01/data-2.csv 30 | * etl-output/02/data-1.csv 31 | * ...etc 32 | * 33 | */ 34 | @FunctionalInterface 35 | public interface S3PartFileKeyGenerator { 36 | @Nonnull 37 | String generateS3Key(int threadNum, int partNumber); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/loader/StringSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.loader; 17 | 18 | import java.util.function.Function; 19 | 20 | /** 21 | * Functional interface for a function that takes an object of a specific type and serializes it into a string. Used 22 | * by loaders that write objects as strings such as S3FastLoader. 23 | * @param The type of object being serialized. 24 | */ 25 | @SuppressWarnings("WeakerAccess") 26 | @FunctionalInterface 27 | public interface StringSerializer extends Function { 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/loader/WrappedLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.loader; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.Loader; 20 | 21 | import javax.annotation.Nullable; 22 | 23 | /** 24 | * Allows a class to masquerade as a Loader by extending this class and providing an implementation of a method to get 25 | * the real Loader it is masquerading as. Used by classes that build complex Loader objects but don't have any real 26 | * implementation themselves such as RedshiftBulkLoader. 27 | * @param The type of object being loaded. 28 | */ 29 | public abstract class WrappedLoader implements Loader { 30 | protected abstract Loader getWrappedLoader(); 31 | 32 | @Override 33 | public void load(T objectToLoad) { 34 | getWrappedLoader().load(objectToLoad); 35 | } 36 | 37 | @Override 38 | public void open(@Nullable EtlMetrics parentMetrics) { 39 | getWrappedLoader().open(parentMetrics); 40 | } 41 | 42 | @Override 43 | public void close() throws Exception { 44 | getWrappedLoader().close(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/lookup/Lookup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.lookup; 17 | 18 | import java.util.Optional; 19 | 20 | /** 21 | * Interface for an object that provides lookup capability across a data-set. Used for random access querying or 22 | * filtering of data in the ETL. Specific implementations may be backed by data structures or wrap service queries. 23 | * @param The class type for keys in this data set. 24 | * @param The class type for values in this data set. 25 | */ 26 | public interface Lookup { 27 | /** 28 | * Attempt to get a specific value from the data set for a given key. 29 | * @param key The key to search the data-set for. 30 | * @return The value stored for the given key, or empty if there was no matching value. 31 | */ 32 | Optional get(KeyType key); 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/lookup/SemaphoreFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.lookup; 17 | 18 | import java.util.concurrent.Semaphore; 19 | 20 | /** 21 | * Simple factory interface for creating Semaphore objects. 22 | */ 23 | public interface SemaphoreFactory { 24 | /** 25 | * Gets a new semaphore object. 26 | * @param numberOfPermits Number of permits to initialize the Semaphore with. 27 | * @return A new Semaphore object. 28 | */ 29 | Semaphore get(int numberOfPermits); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/transformer/FilterTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.Transformer; 20 | import com.amazon.pocketEtl.lookup.Lookup; 21 | import com.google.common.collect.ImmutableList; 22 | 23 | import javax.annotation.Nullable; 24 | import java.util.List; 25 | import java.util.function.BiPredicate; 26 | 27 | /** 28 | * An implementation of transformer that acts as a filter. This type of transformer does not modify the object itself 29 | * therefore only has one generic type that is applied as both the upstream and downstream types. 30 | * @param The type of object being filtered. 31 | */ 32 | public class FilterTransformer implements Transformer { 33 | private final BiPredicate> filter; 34 | private final Lookup filterSetLookup; 35 | 36 | /** 37 | * Standard constructor. 38 | * @param filter A filter used to evaluate whether the object should be filtered. 39 | * @param filterSetLookup A lookup object to be used by the filter to compare against. 40 | */ 41 | public FilterTransformer(BiPredicate> filter, Lookup filterSetLookup) { 42 | this.filter = filter; 43 | this.filterSetLookup = filterSetLookup; 44 | } 45 | 46 | /** 47 | * Filters a single object. 48 | * @param objectToTransform The object to be filtered. 49 | * @return Will return an empty list if the object does not meet the filter criteria, or a list with a single 50 | * instance of the object if it does. 51 | */ 52 | @Override 53 | public List transform(T objectToTransform) { 54 | return filter.test(objectToTransform, filterSetLookup) ? ImmutableList.of(objectToTransform) : ImmutableList.of(); 55 | } 56 | 57 | @Override 58 | public void open(@Nullable EtlMetrics parentMetrics) { 59 | //no-op 60 | } 61 | 62 | @Override 63 | public void close() throws Exception { 64 | //no-op 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/transformer/MapTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer; 17 | 18 | import com.amazon.pocketEtl.Transformer; 19 | import com.google.common.collect.ImmutableList; 20 | import lombok.AccessLevel; 21 | import lombok.RequiredArgsConstructor; 22 | 23 | import java.util.List; 24 | import java.util.function.Function; 25 | 26 | /** 27 | * Simple static constructor for the most basic type of Transformer that maps a single object to another single object, 28 | * effectively a map() operation. 29 | */ 30 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 31 | public class MapTransformer implements Transformer { 32 | private final Function mapFunction; 33 | 34 | /** 35 | * Construct a new transformer that will map a single object to another single object. 36 | * @param mapFunction Lambda function to perform the transformation. 37 | * @param Type of object being mapped. 38 | * @param Type of object being mapped to. 39 | * @return A new Transformer object that can be used in Pocket ETL jobs. 40 | */ 41 | public static MapTransformer of(Function mapFunction) { 42 | return new MapTransformer<>(mapFunction); 43 | } 44 | 45 | @Override 46 | public List transform(T objectToTransform) { 47 | return ImmutableList.of(mapFunction.apply(objectToTransform)); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/pocketEtl/transformer/filter/ContainsFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer.filter; 17 | 18 | import com.amazon.pocketEtl.lookup.Lookup; 19 | 20 | import java.util.function.BiPredicate; 21 | 22 | /** 23 | * Implementation of a filter that tests to see if the filterSet contains an object equal to the one being filtered. 24 | * @param The type of object being filtered. 25 | */ 26 | public class ContainsFilter implements BiPredicate> { 27 | /** 28 | * Tests to see if an equal version of the object can be found in the filterSet. 29 | * @param objectToFilter The object being filtered. 30 | * @param filterSet A lookup containing the values to compare the filtered object against. 31 | * @return true if an object of equivalent value is found in the filterSet; or false if it is not. 32 | */ 33 | @Override 34 | public boolean test(T objectToFilter, Lookup filterSet) { 35 | return filterSet.get(objectToFilter).isPresent(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/EtlCombineStageTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 19 | import com.amazon.pocketEtl.core.producer.EtlProducer; 20 | import com.google.common.collect.ImmutableList; 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.mockito.Mock; 25 | import org.mockito.junit.MockitoJUnitRunner; 26 | 27 | import java.util.Collection; 28 | 29 | import static org.hamcrest.Matchers.contains; 30 | import static org.hamcrest.Matchers.equalTo; 31 | import static org.junit.Assert.assertThat; 32 | import static org.mockito.ArgumentMatchers.any; 33 | import static org.mockito.Mockito.when; 34 | 35 | @RunWith(MockitoJUnitRunner.class) 36 | public class EtlCombineStageTest { 37 | private final static String EXPECTED_DEFAULT_STAGE_NAME = "EtlStream.Combine"; 38 | 39 | @Mock 40 | private EtlStream mockEtlStream1; 41 | @Mock 42 | private EtlStream mockEtlStream2; 43 | @Mock 44 | private EtlStageChain mockEtlStageChain1; 45 | @Mock 46 | private EtlStageChain mockEtlStageChain2; 47 | @Mock 48 | private EtlConsumer mockEtlConsumer; 49 | @Mock 50 | private EtlProducer mockEtlProducer1; 51 | @Mock 52 | private EtlProducer mockEtlProducer2; 53 | @Mock 54 | private EtlProducer mockEtlProducer3; 55 | 56 | private EtlCombineStage etlCombineStage; 57 | 58 | @Before 59 | public void constructEtlCombineStage() { 60 | when(mockEtlStream1.getStageChain()).thenReturn(mockEtlStageChain1); 61 | when(mockEtlStream2.getStageChain()).thenReturn(mockEtlStageChain2); 62 | when(mockEtlStageChain1.constructComponentProducers(any())).thenReturn(ImmutableList.of(mockEtlProducer1)); 63 | when(mockEtlStageChain2.constructComponentProducers(any())).thenReturn(ImmutableList.of(mockEtlProducer2, mockEtlProducer3)); 64 | etlCombineStage = EtlCombineStage.of(ImmutableList.of(mockEtlStream1, mockEtlStream2)); 65 | 66 | } 67 | 68 | @Test 69 | public void staticConstructorSetsPropertiesAsExpected() { 70 | assertThat(etlCombineStage.getStageChains(), equalTo(ImmutableList.of(mockEtlStageChain1, mockEtlStageChain2))); 71 | assertThat(etlCombineStage.getStageName(), equalTo(EXPECTED_DEFAULT_STAGE_NAME)); 72 | } 73 | 74 | @Test 75 | public void combineStreamsCorrectlyFlattensComponentProducers() { 76 | Collection result = etlCombineStage.constructProducersForStage(mockEtlConsumer); 77 | 78 | assertThat(result, contains(mockEtlProducer1, mockEtlProducer2, mockEtlProducer3)); 79 | } 80 | } -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/EtlExtractStageTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 19 | import com.amazon.pocketEtl.core.producer.EtlProducer; 20 | import com.amazon.pocketEtl.core.producer.EtlProducerFactory; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.mockito.Mock; 25 | import org.mockito.junit.MockitoJUnitRunner; 26 | 27 | import java.util.Collection; 28 | import java.util.Collections; 29 | 30 | import static org.hamcrest.Matchers.contains; 31 | import static org.hamcrest.Matchers.equalTo; 32 | import static org.junit.Assert.assertThat; 33 | import static org.mockito.ArgumentMatchers.any; 34 | import static org.mockito.ArgumentMatchers.anyString; 35 | import static org.mockito.Mockito.times; 36 | import static org.mockito.Mockito.verify; 37 | import static org.mockito.Mockito.when; 38 | 39 | @RunWith(MockitoJUnitRunner.class) 40 | public class EtlExtractStageTest { 41 | private static String EXPECTED_DEFAULT_NAME = "EtlStream.Extract"; 42 | 43 | @Mock 44 | private Extractor mockExtractor; 45 | @Mock 46 | private EtlProducerFactory mockEtlProducerFactory; 47 | @Mock 48 | private EtlProducer mockEtlProducer; 49 | @Mock 50 | private EtlConsumer mockEtlConsumer; 51 | 52 | @Test 53 | public void staticConstructorWithSingleExtractor() { 54 | EtlExtractStage etlExtractStage = EtlExtractStage.of(mockExtractor); 55 | 56 | assertThat(etlExtractStage.getExtractors(), equalTo(Collections.singletonList(mockExtractor))); 57 | assertThat(etlExtractStage.getStageName(), equalTo(EXPECTED_DEFAULT_NAME)); 58 | } 59 | 60 | @Test 61 | public void staticConstructorWithMultipleExtractors() { 62 | EtlExtractStage etlExtractStage = EtlExtractStage.of(ImmutableList.of(mockExtractor, mockExtractor)); 63 | 64 | assertThat(etlExtractStage.getExtractors(), equalTo(ImmutableList.of(mockExtractor, mockExtractor))); 65 | assertThat(etlExtractStage.getStageName(), equalTo(EXPECTED_DEFAULT_NAME)); 66 | } 67 | 68 | @Test 69 | public void withNameOverridesName() { 70 | String customName = "custom-name"; 71 | EtlExtractStage etlExtractStage = EtlExtractStage.of(mockExtractor).withName(customName); 72 | 73 | assertThat(etlExtractStage.getStageName(), equalTo(customName)); 74 | } 75 | 76 | @Test 77 | public void constructProducersForStageUsesFactory() { 78 | EtlExtractStage etlExtractStage = new EtlExtractStage(ImmutableList.of(mockExtractor, mockExtractor), 79 | EXPECTED_DEFAULT_NAME, mockEtlProducerFactory); 80 | when(mockEtlProducerFactory.newExtractorProducer(anyString(), any(), any())).thenReturn(mockEtlProducer); 81 | 82 | Collection result = etlExtractStage.constructProducersForStage(mockEtlConsumer); 83 | 84 | assertThat(result, contains(mockEtlProducer, mockEtlProducer)); 85 | verify(mockEtlProducerFactory, times(2)).newExtractorProducer(EXPECTED_DEFAULT_NAME, mockExtractor, mockEtlConsumer); 86 | } 87 | } -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/EtlProfilingScopeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import static org.mockito.ArgumentMatchers.anyDouble; 19 | import static org.mockito.ArgumentMatchers.anyString; 20 | import static org.mockito.ArgumentMatchers.eq; 21 | import static org.mockito.Mockito.never; 22 | import static org.mockito.Mockito.verify; 23 | import static org.mockito.Mockito.when; 24 | 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import org.mockito.Mock; 29 | import org.mockito.Mockito; 30 | import org.mockito.junit.MockitoJUnitRunner; 31 | 32 | @RunWith(MockitoJUnitRunner.class) 33 | public class EtlProfilingScopeTest { 34 | private static final String SCOPE_NAME = "scope-name"; 35 | private static final String COUNTER_NAME = "counter-name"; 36 | 37 | @Mock 38 | private EtlMetrics mockMetrics; 39 | 40 | @Mock 41 | private EtlMetrics mockChildMetrics; 42 | 43 | private EtlProfilingScope etlProfilingScope; 44 | 45 | @Before 46 | public void stubMetricsAndCreateScope() { 47 | when(mockMetrics.createChildMetrics()).thenReturn(mockChildMetrics); 48 | etlProfilingScope = new EtlProfilingScope(mockMetrics, SCOPE_NAME); 49 | verify(mockChildMetrics, never()).addTime(anyString(), anyDouble()); 50 | } 51 | 52 | @Test 53 | public void close_emitsTime() { 54 | etlProfilingScope.close(); 55 | 56 | verify(mockChildMetrics).addTime(eq(SCOPE_NAME), anyDouble()); 57 | } 58 | 59 | @Test 60 | public void close_afterClose_doesNotEmitsTime() { 61 | etlProfilingScope.close(); 62 | Mockito.reset(mockChildMetrics); 63 | 64 | etlProfilingScope.close(); 65 | 66 | verify(mockChildMetrics, never()).addTime(anyString(), anyDouble()); 67 | } 68 | 69 | @Test 70 | public void addCounter_emitsCount() { 71 | etlProfilingScope.addCounter(COUNTER_NAME, 123); 72 | 73 | verify(mockChildMetrics).addCount(COUNTER_NAME, 123.0); 74 | } 75 | 76 | @Test 77 | public void addCounter_doesNotEmitCountAfterClose() { 78 | etlProfilingScope.close(); 79 | etlProfilingScope.addCounter(COUNTER_NAME, 123); 80 | 81 | verify(mockChildMetrics, never()).addCount(anyString(), anyDouble()); 82 | } 83 | } -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/EtlTestBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Data; 20 | import lombok.NoArgsConstructor; 21 | import org.junit.After; 22 | import org.junit.Before; 23 | 24 | import static org.mockito.Mockito.mock; 25 | import static org.mockito.Mockito.when; 26 | 27 | public abstract class EtlTestBase { 28 | protected EtlProfilingScope etlProfilingScope; 29 | protected EtlMetrics mockMetrics; 30 | 31 | @Data 32 | @NoArgsConstructor 33 | @AllArgsConstructor 34 | protected static class TestDTO { 35 | private String value; 36 | } 37 | 38 | @Before 39 | public void initializeServiceLogEntry() { 40 | mockMetrics = mock(EtlMetrics.class); 41 | associateMockMetricsProviders(mockMetrics, mockMetrics); 42 | etlProfilingScope = new EtlProfilingScope(mockMetrics, "UnitTest"); 43 | } 44 | 45 | private void associateMockMetricsProviders(EtlMetrics mockParentMetrics, EtlMetrics mockChildMetrics) { 46 | when(mockParentMetrics.createChildMetrics()).thenReturn(mockChildMetrics); 47 | } 48 | 49 | @After 50 | public void teardownServiceLogEntry() { 51 | etlProfilingScope.close(); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/EtlConsumerFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.Loader; 19 | import com.amazon.pocketEtl.Transformer; 20 | import com.amazon.pocketEtl.core.DefaultLoggingStrategy; 21 | import com.amazon.pocketEtl.core.executor.EtlExecutor; 22 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 23 | import org.apache.logging.log4j.Logger; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | import org.junit.runner.RunWith; 27 | import org.mockito.Mock; 28 | import org.mockito.junit.MockitoJUnitRunner; 29 | 30 | import static org.hamcrest.Matchers.instanceOf; 31 | import static org.junit.Assert.assertThat; 32 | import static org.mockito.Mockito.when; 33 | 34 | @RunWith(MockitoJUnitRunner.class) 35 | public class EtlConsumerFactoryTest { 36 | private static final String STAGE_NAME = "test-stage"; 37 | 38 | @Mock 39 | private Loader mockLoader; 40 | @Mock 41 | private Transformer mockTransformer; 42 | @Mock 43 | private EtlConsumer mockErrorConsumer; 44 | @Mock 45 | private EtlConsumer mockDownstreamConsumer; 46 | @Mock 47 | private EtlExecutor mockEtlExecutor; 48 | @Mock 49 | private Logger mockLogger; 50 | @Mock 51 | private EtlExecutorFactory mockEtlExecutorFactory; 52 | 53 | private EtlConsumerFactory etlConsumerFactory; 54 | 55 | @Before 56 | public void initializeConsumerFactory() { 57 | etlConsumerFactory = new EtlConsumerFactory(mockEtlExecutorFactory); 58 | when(mockEtlExecutorFactory.newImmediateExecutionEtlExecutor()).thenReturn(mockEtlExecutor); 59 | } 60 | 61 | @Test 62 | public void newLoaderCreatesAWrappedLoaderConsumer() { 63 | EtlConsumer consumer = etlConsumerFactory.newLoader(STAGE_NAME, mockLoader, Object.class, mockErrorConsumer, mockEtlExecutor); 64 | verifyWrappedConsumerStack(consumer, LoaderEtlConsumer.class); 65 | } 66 | 67 | @Test 68 | public void newTransformerCreatesAWrappedTransformerConsumer() { 69 | EtlConsumer consumer = etlConsumerFactory.newTransformer(STAGE_NAME, mockTransformer, Object.class, mockDownstreamConsumer, 70 | mockErrorConsumer, mockEtlExecutor); 71 | 72 | verifyWrappedConsumerStack(consumer, TransformerEtlConsumer.class); 73 | } 74 | 75 | @Test 76 | public void newLogAsErrorCreatesAWrappedLogAsErrorConsumer() { 77 | EtlConsumer consumer = etlConsumerFactory.newLogAsErrorConsumer(STAGE_NAME, mockLogger, Object.class, new DefaultLoggingStrategy<>()); 78 | 79 | verifyWrappedConsumerStack(consumer, LogAsErrorEtlConsumer.class); 80 | } 81 | 82 | private void verifyWrappedConsumerStack(EtlConsumer consumer, Class expectedClass) { 83 | assertThat(consumer, instanceOf(SmartEtlConsumer.class)); 84 | 85 | consumer = ((SmartEtlConsumer)consumer).getWrappedEtlConsumer(); 86 | assertThat(consumer, instanceOf(MetricsEmissionEtlConsumer.class)); 87 | 88 | consumer = ((MetricsEmissionEtlConsumer)consumer).getDownstreamEtlConsumer(); 89 | assertThat(consumer, instanceOf(ExecutorEtlConsumer.class)); 90 | 91 | consumer = ((ExecutorEtlConsumer)consumer).getWrappedEtlConsumer(); 92 | assertThat(consumer, instanceOf(expectedClass)); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/ExecutorEtlConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.EtlTestBase; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import com.amazon.pocketEtl.core.executor.EtlExecutor; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.mockito.Mock; 26 | import org.mockito.junit.MockitoJUnitRunner; 27 | 28 | import static org.mockito.ArgumentMatchers.any; 29 | import static org.mockito.ArgumentMatchers.eq; 30 | import static org.mockito.Mockito.doAnswer; 31 | import static org.mockito.Mockito.times; 32 | import static org.mockito.Mockito.verify; 33 | import static org.mockito.Mockito.when; 34 | 35 | @RunWith(MockitoJUnitRunner.class) 36 | public class ExecutorEtlConsumerTest extends EtlTestBase { 37 | private static final String TEST_NAME = "TestName"; 38 | 39 | @Mock 40 | private EtlStreamObject mockEtlStreamObject; 41 | 42 | @Mock 43 | private EtlExecutor mockEtlExecutor; 44 | 45 | @Mock 46 | private EtlConsumer mockEtlConsumer; 47 | 48 | private ExecutorEtlConsumer executorConsumer; 49 | 50 | @Before 51 | public void constructTransformer() { 52 | executorConsumer = new ExecutorEtlConsumer(TEST_NAME, mockEtlConsumer, mockEtlExecutor); 53 | } 54 | 55 | @Before 56 | public void initializeMockExecutor() { 57 | when(mockEtlExecutor.isShutdown()).thenReturn(false); 58 | } 59 | 60 | @Test 61 | public void consumeSubmitsARunnableThatWritesToConsumer() { 62 | doAnswer(invocation -> { 63 | Runnable runnable = (Runnable) invocation.getArguments()[0]; 64 | runnable.run(); 65 | return null; 66 | }).when(mockEtlExecutor).submit(any(Runnable.class), any(EtlMetrics.class)); 67 | 68 | executorConsumer.open(etlProfilingScope.getMetrics()); 69 | executorConsumer.consume(mockEtlStreamObject); 70 | 71 | verify(mockEtlExecutor, times(1)).submit(any(Runnable.class), eq(etlProfilingScope.getMetrics())); 72 | verify(mockEtlConsumer, times(1)).consume(eq(mockEtlStreamObject)); 73 | } 74 | 75 | @Test 76 | public void closeShutsDownExecutor() throws Exception { 77 | executorConsumer.open(mockMetrics); 78 | executorConsumer.close(); 79 | 80 | verify(mockEtlExecutor).shutdown(); 81 | } 82 | 83 | @Test 84 | public void closeClosesConsumer() throws Exception { 85 | executorConsumer.open(mockMetrics); 86 | executorConsumer.close(); 87 | 88 | verify(mockEtlConsumer).close(); 89 | } 90 | 91 | @Test 92 | public void openOpensConsumer() { 93 | executorConsumer.open(etlProfilingScope.getMetrics()); 94 | 95 | verify(mockEtlConsumer).open(eq(etlProfilingScope.getMetrics())); 96 | } 97 | 98 | 99 | @Test(expected = IllegalStateException.class) 100 | public void consumeThrowsIllegalStateExceptionIfExecutorServiceIsShutdown() { 101 | when(mockEtlExecutor.isShutdown()).thenReturn(true); 102 | executorConsumer.consume(mockEtlStreamObject); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/LoaderEtlConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import com.amazon.pocketEtl.Loader; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.mockito.Mock; 25 | import org.mockito.junit.MockitoJUnitRunner; 26 | 27 | import static org.mockito.ArgumentMatchers.any; 28 | import static org.mockito.ArgumentMatchers.eq; 29 | import static org.mockito.Mockito.doThrow; 30 | import static org.mockito.Mockito.times; 31 | import static org.mockito.Mockito.verify; 32 | import static org.mockito.Mockito.when; 33 | 34 | @RunWith(MockitoJUnitRunner.class) 35 | public class LoaderEtlConsumerTest extends EtlTestBase { 36 | private static final String TEST_NAME = "TestName"; 37 | 38 | @Mock 39 | private Loader mockLoader; 40 | 41 | @Mock 42 | private EtlConsumer mockErrorEtlConsumer; 43 | 44 | @Mock 45 | private EtlStreamObject mockEtlStreamObject; 46 | 47 | @Mock 48 | private TestDTO mockTestDTO; 49 | 50 | private LoaderEtlConsumer loaderConsumer; 51 | 52 | @Before 53 | public void constructWorker() { 54 | when(mockEtlStreamObject.get(any())).thenReturn(mockTestDTO); 55 | loaderConsumer = new LoaderEtlConsumer<>(TEST_NAME, mockLoader, TestDTO.class, mockErrorEtlConsumer); 56 | } 57 | 58 | @Test 59 | public void consumeLoadsASingleObject() { 60 | loaderConsumer.open(mockMetrics); 61 | loaderConsumer.consume(mockEtlStreamObject); 62 | 63 | verify(mockLoader, times(1)).load(eq(mockTestDTO)); 64 | } 65 | 66 | @Test 67 | public void closeClosesLoader() throws Exception { 68 | loaderConsumer.open(mockMetrics); 69 | loaderConsumer.close(); 70 | 71 | verify(mockLoader).close(); 72 | } 73 | 74 | @Test 75 | public void closeClosesErrorConsumer() throws Exception { 76 | loaderConsumer.open(mockMetrics); 77 | loaderConsumer.close(); 78 | 79 | verify(mockErrorEtlConsumer).close(); 80 | } 81 | 82 | @Test 83 | public void closeClosesErrorConsumerEvenAfterARuntimeException() throws Exception { 84 | doThrow(new RuntimeException("Test exception")).when(mockLoader).close(); 85 | loaderConsumer.open(mockMetrics); 86 | loaderConsumer.close(); 87 | 88 | verify(mockErrorEtlConsumer).close(); 89 | } 90 | 91 | @Test 92 | public void openOpensLoader() throws Exception { 93 | loaderConsumer.open(etlProfilingScope.getMetrics()); 94 | 95 | verify(mockLoader).open(eq(etlProfilingScope.getMetrics())); 96 | } 97 | 98 | @Test 99 | public void openOpensErrorConsumer() throws Exception { 100 | loaderConsumer.open(etlProfilingScope.getMetrics()); 101 | 102 | verify(mockErrorEtlConsumer).open(eq(etlProfilingScope.getMetrics())); 103 | } 104 | 105 | @Test 106 | public void consumePassesToTheErrorConsumerOnRuntimeException() { 107 | doThrow(new RuntimeException("test")).when(mockLoader).load(any(TestDTO.class)); 108 | 109 | loaderConsumer.open(mockMetrics); 110 | loaderConsumer.consume(mockEtlStreamObject); 111 | 112 | verify(mockErrorEtlConsumer).consume(mockEtlStreamObject); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/LogAsErrorEtlConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import com.amazon.pocketEtl.core.DefaultLoggingStrategy; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import org.apache.logging.log4j.Logger; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.mockito.Mock; 26 | import org.mockito.junit.MockitoJUnitRunner; 27 | 28 | import java.util.function.Function; 29 | 30 | import static org.mockito.ArgumentMatchers.any; 31 | import static org.mockito.ArgumentMatchers.contains; 32 | import static org.mockito.ArgumentMatchers.endsWith; 33 | import static org.mockito.ArgumentMatchers.eq; 34 | import static org.mockito.Mockito.mock; 35 | import static org.mockito.Mockito.verify; 36 | import static org.mockito.Mockito.verifyNoMoreInteractions; 37 | import static org.mockito.Mockito.when; 38 | 39 | @RunWith(MockitoJUnitRunner.class) 40 | public class LogAsErrorEtlConsumerTest extends EtlTestBase { 41 | private static final String TEST_NAME = "TestName"; 42 | private static final String TEST_LOGGING_STRATEGY_STRING = "TestLoggingStrategyString"; 43 | 44 | @Mock 45 | private Logger mockLogger; 46 | 47 | @Mock 48 | private EtlStreamObject mockEtlStreamObject; 49 | 50 | @Mock 51 | private Function mockLoggingStrategy; 52 | 53 | private LogAsErrorEtlConsumer logAsErrorEtlConsumer; 54 | 55 | private LogAsErrorEtlConsumer logAsErrorConsumerWithLogging; 56 | 57 | @Before 58 | public void constructLogAsErrorConsumerWithLoggingStrategy() { 59 | logAsErrorConsumerWithLogging = new LogAsErrorEtlConsumer<>(TEST_NAME, mockLogger, TestDTO.class, mockLoggingStrategy); 60 | } 61 | 62 | @Before 63 | public void constructLogAsErrorConsumer() { 64 | logAsErrorEtlConsumer = new LogAsErrorEtlConsumer<>(TEST_NAME, mockLogger, TestDTO.class, new DefaultLoggingStrategy<>()); 65 | } 66 | 67 | @Before 68 | public void stubLoggingExecute() { 69 | when(mockLoggingStrategy.apply(any())).thenReturn(TEST_LOGGING_STRATEGY_STRING); 70 | } 71 | 72 | @Test 73 | public void consumeDoesNotRevealObjectWithDefaultLoggingStrategy() { 74 | Object mockLogObject = mock(Object.class); 75 | when(mockEtlStreamObject.get(any())).thenReturn(mockLogObject); 76 | 77 | logAsErrorEtlConsumer.open(mockMetrics); 78 | logAsErrorEtlConsumer.consume(mockEtlStreamObject); 79 | 80 | verifyNoMoreInteractions(mockLogObject); 81 | } 82 | 83 | @Test 84 | public void consumeLogsErrorWithLoggingStrategy() { 85 | logAsErrorConsumerWithLogging.open(mockMetrics); 86 | logAsErrorConsumerWithLogging.consume(mockEtlStreamObject); 87 | 88 | verify(mockLogger).error(endsWith(TEST_LOGGING_STRATEGY_STRING)); 89 | } 90 | 91 | @Test 92 | public void openDoesNothing() { 93 | logAsErrorEtlConsumer.open(etlProfilingScope.getMetrics()); 94 | 95 | verifyNoMoreInteractions(mockLogger); 96 | } 97 | 98 | @Test 99 | public void closeDoesNothing() throws Exception { 100 | logAsErrorEtlConsumer.open(mockMetrics); 101 | logAsErrorEtlConsumer.close(); 102 | 103 | verifyNoMoreInteractions(mockLogger); 104 | } 105 | 106 | @Test 107 | public void consumeStillLogsAnErrorIfStrategyThrowsRuntimeException() { 108 | RuntimeException e = new RuntimeException("Test exception"); 109 | 110 | when(mockLoggingStrategy.apply(any())).thenThrow(e); 111 | 112 | logAsErrorConsumerWithLogging.open(mockMetrics); 113 | logAsErrorConsumerWithLogging.consume(mockEtlStreamObject); 114 | 115 | verify(mockLogger).error(contains(mockEtlStreamObject.getClass().getSimpleName()), eq(e)); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/MetricsEmissionEtlConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlProfilingScope; 19 | import com.amazon.pocketEtl.EtlTestBase; 20 | import com.amazon.pocketEtl.core.EtlStreamObject; 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.mockito.Mock; 25 | import org.mockito.junit.MockitoJUnitRunner; 26 | 27 | import static org.mockito.ArgumentMatchers.eq; 28 | import static org.mockito.Mockito.times; 29 | import static org.mockito.Mockito.verify; 30 | 31 | @RunWith(MockitoJUnitRunner.class) 32 | public class MetricsEmissionEtlConsumerTest extends EtlTestBase { 33 | private final static String STAGE_NAME = "testStage"; 34 | private final static String JOB_NAME = "jobName"; 35 | 36 | @Mock 37 | private EtlStreamObject mockEtlStreamObject; 38 | 39 | @Mock 40 | private EtlConsumer mockDownstreamEtlConsumer; 41 | 42 | @Mock 43 | private EtlProfilingScope mockEtlProfilingScope; 44 | 45 | private MetricsEmissionEtlConsumer metricsEmissionConsumer; 46 | 47 | 48 | @Before 49 | public void createMetricsEmissionConsumer() { 50 | metricsEmissionConsumer = new MetricsEmissionEtlConsumer(JOB_NAME + "." + STAGE_NAME, mockDownstreamEtlConsumer); 51 | } 52 | 53 | @Test 54 | public void consumeEmitsACounterForEveryRecordConsumed() { 55 | metricsEmissionConsumer.open(mockMetrics); 56 | metricsEmissionConsumer.consume(mockEtlStreamObject); 57 | metricsEmissionConsumer.consume(mockEtlStreamObject); 58 | metricsEmissionConsumer.consume(mockEtlStreamObject); 59 | 60 | verify(mockMetrics, times(3)) 61 | .addCount(eq(JOB_NAME + "." + STAGE_NAME + ".recordsProcessed"), eq(1.0)); 62 | } 63 | 64 | @Test 65 | public void consumeEmitsAZeroCounterWithoutAnyRecordConsumed() { 66 | metricsEmissionConsumer.open(mockMetrics); 67 | 68 | verify(mockMetrics).addCount(eq(JOB_NAME + "." + STAGE_NAME + ".recordsProcessed"), eq(0.0)); 69 | } 70 | 71 | @Test 72 | public void consumePassesRecordToDownstreamConsumer() { 73 | metricsEmissionConsumer.open(mockMetrics); 74 | metricsEmissionConsumer.consume(mockEtlStreamObject); 75 | 76 | verify(mockDownstreamEtlConsumer, times(1)).consume(eq(mockEtlStreamObject)); 77 | } 78 | 79 | @Test 80 | public void openCallsOpenOnDownstreamConsumer() { 81 | metricsEmissionConsumer.open(mockEtlProfilingScope.getMetrics()); 82 | 83 | verify(mockDownstreamEtlConsumer, times(1)).open(mockEtlProfilingScope.getMetrics()); 84 | } 85 | 86 | @Test 87 | public void closeCallsCloseOnDownstreamConsumer() throws Exception { 88 | metricsEmissionConsumer.close(); 89 | 90 | verify(mockDownstreamEtlConsumer, times(1)).close(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/consumer/SmartEtlConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.consumer; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import com.amazon.pocketEtl.core.EtlStreamObject; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.mockito.Mock; 24 | import org.mockito.junit.MockitoJUnitRunner; 25 | 26 | import static org.mockito.ArgumentMatchers.eq; 27 | import static org.mockito.Mockito.never; 28 | import static org.mockito.Mockito.times; 29 | import static org.mockito.Mockito.verify; 30 | 31 | @RunWith(MockitoJUnitRunner.class) 32 | public class SmartEtlConsumerTest extends EtlTestBase { 33 | private final static String TEST_NAME = "TestName"; 34 | 35 | @Mock 36 | private EtlConsumer mockWrappedEtlConsumer; 37 | 38 | @Mock 39 | private EtlStreamObject mockEtlStreamObject; 40 | 41 | private SmartEtlConsumer smartConsumer; 42 | 43 | @Before 44 | public void constructSmartTrackingConsumer() { 45 | smartConsumer = new SmartEtlConsumer(TEST_NAME, mockWrappedEtlConsumer); 46 | } 47 | 48 | @Test 49 | public void openOpensWrappedConsumer() { 50 | smartConsumer.open(etlProfilingScope.getMetrics()); 51 | 52 | verify(mockWrappedEtlConsumer).open(eq(etlProfilingScope.getMetrics())); 53 | } 54 | 55 | @Test 56 | public void openOpensWrappedConsumerOnlyOnceWhenCalledMultipleTimes() { 57 | smartConsumer.open(etlProfilingScope.getMetrics()); 58 | smartConsumer.open(etlProfilingScope.getMetrics()); 59 | smartConsumer.open(etlProfilingScope.getMetrics()); 60 | 61 | verify(mockWrappedEtlConsumer, times(1)).open(eq(etlProfilingScope.getMetrics())); 62 | } 63 | 64 | @Test 65 | public void closeClosesWrappedConsumer() throws Exception { 66 | smartConsumer.open(etlProfilingScope.getMetrics()); 67 | smartConsumer.close(); 68 | 69 | verify(mockWrappedEtlConsumer).close(); 70 | } 71 | 72 | @Test 73 | public void consumePassesObjectToWrappedConsumer() { 74 | smartConsumer.open(mockMetrics); 75 | smartConsumer.consume(mockEtlStreamObject); 76 | 77 | verify(mockWrappedEtlConsumer).consume(eq(mockEtlStreamObject)); 78 | } 79 | 80 | @Test 81 | public void closeOnlyClosesWhenOpenCountHasBeenReached() throws Exception { 82 | smartConsumer.open(etlProfilingScope.getMetrics()); 83 | smartConsumer.open(etlProfilingScope.getMetrics()); 84 | smartConsumer.open(etlProfilingScope.getMetrics()); 85 | smartConsumer.close(); 86 | smartConsumer.close(); 87 | verify(mockWrappedEtlConsumer, never()).close(); 88 | 89 | smartConsumer.close(); 90 | verify(mockWrappedEtlConsumer, times(1)).close(); 91 | } 92 | 93 | @Test(expected = IllegalStateException.class) 94 | public void attemptingToCloseTooManyTimesThrowsIllegalStateException() throws Exception { 95 | smartConsumer.open(etlProfilingScope.getMetrics()); 96 | smartConsumer.close(); 97 | smartConsumer.close(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/executor/EtlExecutorFactoryUnboundFixedThreadsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | import org.junit.After; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.mockito.Mock; 23 | import org.mockito.junit.MockitoJUnitRunner; 24 | 25 | import java.util.concurrent.BlockingQueue; 26 | import java.util.concurrent.LinkedBlockingQueue; 27 | import java.util.concurrent.RejectedExecutionException; 28 | import java.util.concurrent.RejectedExecutionHandler; 29 | import java.util.concurrent.ThreadPoolExecutor; 30 | import java.util.concurrent.atomic.AtomicInteger; 31 | import java.util.stream.IntStream; 32 | 33 | import static org.hamcrest.Matchers.equalTo; 34 | import static org.hamcrest.Matchers.instanceOf; 35 | import static org.hamcrest.Matchers.is; 36 | import static org.junit.Assert.assertThat; 37 | 38 | @RunWith(MockitoJUnitRunner.class) 39 | public class EtlExecutorFactoryUnboundFixedThreadsTest { 40 | private final static int NUMBER_OF_WORKERS = 3; 41 | 42 | @Mock 43 | private ThreadPoolExecutor mockThreadPoolExecutor; 44 | 45 | @Mock 46 | private Runnable mockRunnable; 47 | 48 | private EtlExecutorFactory etlExecutorFactory = new EtlExecutorFactory(); 49 | private ThreadPoolExecutor realThreadPoolExecutor; 50 | private RejectedExecutionHandler rejectedExecutionHandler; 51 | private EtlExecutor etlExecutor; 52 | 53 | @Before 54 | public void constructEtlExecutor() { 55 | etlExecutor = etlExecutorFactory.newUnboundFixedThreadsEtlExecutorFactory(NUMBER_OF_WORKERS); 56 | realThreadPoolExecutor = (ThreadPoolExecutor) ((ExecutorServiceEtlExecutor) etlExecutor).getExecutorService(); 57 | rejectedExecutionHandler = realThreadPoolExecutor.getRejectedExecutionHandler(); 58 | } 59 | 60 | @After 61 | public void teardownEtlExecutor() throws Exception { 62 | etlExecutor.shutdown(); 63 | } 64 | 65 | @Test(expected = RejectedExecutionException.class) 66 | public void rejectedExecutionThrowsRejectedExecutionException() { 67 | rejectedExecutionHandler.rejectedExecution(mockRunnable, mockThreadPoolExecutor); 68 | } 69 | 70 | @Test 71 | public void threadPoolExecutorHasCorrectCorePoolSize() { 72 | assertThat(realThreadPoolExecutor.getCorePoolSize(), is(NUMBER_OF_WORKERS)); 73 | } 74 | 75 | @Test 76 | public void threadPoolExecutorHasCorrectMaxPoolSize() { 77 | assertThat(realThreadPoolExecutor.getMaximumPoolSize(), is(NUMBER_OF_WORKERS)); 78 | } 79 | 80 | @Test 81 | public void threadPoolExecutorQueueIsCorrectType() { 82 | BlockingQueue queue = realThreadPoolExecutor.getQueue(); 83 | assertThat(queue, instanceOf(LinkedBlockingQueue.class)); 84 | } 85 | 86 | @Test 87 | public void executorCanDoRealWork() throws Exception { 88 | AtomicInteger workCounter = new AtomicInteger(0); 89 | 90 | IntStream.range(0, 100).forEach(i -> etlExecutor.submit(workCounter::incrementAndGet, null)); 91 | etlExecutor.shutdown(); 92 | 93 | assertThat(workCounter.get(), equalTo(100)); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/executor/ImmediateExecutionEtlExecutorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.executor; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import org.junit.Test; 20 | 21 | import java.util.concurrent.RejectedExecutionException; 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | import java.util.stream.IntStream; 24 | 25 | import static org.hamcrest.Matchers.equalTo; 26 | import static org.hamcrest.core.Is.is; 27 | import static org.junit.Assert.assertThat; 28 | import static org.mockito.ArgumentMatchers.anyDouble; 29 | import static org.mockito.ArgumentMatchers.eq; 30 | import static org.mockito.Mockito.verify; 31 | 32 | public class ImmediateExecutionEtlExecutorTest extends EtlTestBase { 33 | private final ImmediateExecutionEtlExecutor etlExecutor = new ImmediateExecutionEtlExecutor(); 34 | 35 | @Test 36 | public void executorCanDoRealWork() throws Exception { 37 | AtomicInteger workCounter = new AtomicInteger(0); 38 | 39 | IntStream.range(0, 100).forEach(i -> etlExecutor.submit(workCounter::incrementAndGet, etlProfilingScope.getMetrics())); 40 | etlExecutor.shutdown(); 41 | 42 | assertThat(workCounter.get(), equalTo(100)); 43 | } 44 | 45 | @Test 46 | public void verifyIsShutdownIsFalseByDefault() { 47 | boolean isShutdown = etlExecutor.isShutdown(); 48 | 49 | assertThat(isShutdown, is(false)); 50 | } 51 | 52 | @Test 53 | public void verifyIsShutdownIsTrueAfterShutdown() throws Exception { 54 | etlExecutor.shutdown(); 55 | boolean isShutdown = etlExecutor.isShutdown(); 56 | 57 | assertThat(isShutdown, is(true)); 58 | } 59 | 60 | @Test(expected = RejectedExecutionException.class) 61 | public void submitAfterShutdownThrowsIllegalStateException() throws Exception { 62 | etlExecutor.shutdown(); 63 | etlExecutor.submit(() -> {}, etlProfilingScope.getMetrics()); 64 | } 65 | 66 | @Test 67 | public void submitSwallowsException() { 68 | etlExecutor.submit(() -> { 69 | throw new RuntimeException("Test Exception"); 70 | }, etlProfilingScope.getMetrics()); 71 | } 72 | 73 | @Test 74 | public void submitWrapsExecutionInServiceLogEntryScope() { 75 | etlExecutor.submit(() -> {}, etlProfilingScope.getMetrics()); 76 | 77 | verify(mockMetrics).addTime(eq("SingleThreadedEtlExecutor.submit"), anyDouble()); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/core/producer/EtlProducerFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.core.producer; 17 | 18 | import com.amazon.pocketEtl.Extractor; 19 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 20 | import com.amazon.pocketEtl.core.executor.EtlExecutorFactory; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.mockito.Mock; 26 | import org.mockito.junit.MockitoJUnitRunner; 27 | 28 | import static org.hamcrest.Matchers.instanceOf; 29 | import static org.junit.Assert.assertThat; 30 | import static org.mockito.Mockito.verify; 31 | 32 | @RunWith(MockitoJUnitRunner.class) 33 | public class EtlProducerFactoryTest { 34 | private static final String PRODUCER_NAME = "producer-name"; 35 | 36 | @Mock 37 | private Extractor mockExtractor; 38 | @Mock 39 | private EtlConsumer mockEtlConsumer; 40 | @Mock 41 | private EtlProducer mockEtlProducer; 42 | @Mock 43 | private EtlExecutorFactory mockEtlExecutorFactory; 44 | 45 | private EtlProducerFactory etlProducerFactory; 46 | 47 | @Before 48 | public void initializeEtlProducerFactory() { 49 | etlProducerFactory = new EtlProducerFactory(mockEtlExecutorFactory); 50 | } 51 | 52 | @Test 53 | public void newExtractorProducerReturnsCorrectType() { 54 | EtlProducer producer = etlProducerFactory.newExtractorProducer(PRODUCER_NAME, mockExtractor, mockEtlConsumer); 55 | 56 | assertThat(producer, instanceOf(ExtractorEtlProducer.class)); 57 | } 58 | 59 | @Test 60 | public void combineProducersReturnsCorrectType() { 61 | EtlProducer producer = etlProducerFactory.combineProducers(PRODUCER_NAME, ImmutableList.of(mockEtlProducer), 1); 62 | 63 | assertThat(producer, instanceOf(ExecutorEtlProducer.class)); 64 | } 65 | 66 | @Test 67 | public void combineProducersUsesCorrectExecutor() { 68 | etlProducerFactory.combineProducers(PRODUCER_NAME, ImmutableList.of(mockEtlProducer), 5); 69 | verify(mockEtlExecutorFactory).newUnboundFixedThreadsEtlExecutorFactory(5); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/extractor/InputStreamExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import org.junit.Before; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.mockito.Mock; 22 | import org.mockito.junit.MockitoJUnitRunner; 23 | 24 | import java.io.InputStream; 25 | import java.util.Iterator; 26 | import java.util.Optional; 27 | 28 | import static org.hamcrest.Matchers.equalTo; 29 | import static org.junit.Assert.assertThat; 30 | import static org.mockito.Mockito.verify; 31 | import static org.mockito.Mockito.when; 32 | 33 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 34 | 35 | @RunWith(MockitoJUnitRunner.class) 36 | public class InputStreamExtractorTest { 37 | @Mock 38 | private InputStream mockInputStream; 39 | 40 | @Mock 41 | private Iterator mockIterator; 42 | 43 | @Mock 44 | private InputStreamMapper mockInputStreamMapper; 45 | 46 | private InputStreamExtractor inputStreamExtractor; 47 | 48 | @Before 49 | public void initializeMocks() { 50 | when(mockInputStreamMapper.apply(mockInputStream)).thenReturn(mockIterator); 51 | 52 | inputStreamExtractor = 53 | (InputStreamExtractor)InputStreamExtractor.of(() -> mockInputStream, mockInputStreamMapper); 54 | } 55 | 56 | @Test 57 | public void extractorCanExtractValues() { 58 | when(mockIterator.hasNext()).thenReturn(true, true, true, false); 59 | when(mockIterator.next()).thenReturn("one", "two", "three").thenThrow(new RuntimeException("No more values")); 60 | 61 | inputStreamExtractor.open(null); 62 | assertThat(inputStreamExtractor.next(), equalTo(Optional.of("one"))); 63 | assertThat(inputStreamExtractor.next(), equalTo(Optional.of("two"))); 64 | assertThat(inputStreamExtractor.next(), equalTo(Optional.of("three"))); 65 | assertThat(inputStreamExtractor.next(), equalTo(Optional.empty())); 66 | } 67 | 68 | @Test(expected = UnrecoverableStreamFailureException.class) 69 | public void nextThrowsUnrecoverableStreamFailureExceptionIfIOExceptionOccursWhileReadingStream() { 70 | when(mockIterator.hasNext()).thenReturn(true, true, true, false); 71 | when(mockIterator.next()).thenThrow(new RuntimeException("Boom")); 72 | 73 | inputStreamExtractor.open(null); 74 | inputStreamExtractor.next(); 75 | } 76 | 77 | @Test(expected = IllegalStateException.class) 78 | public void nextThrowsIllegalStateExceptionIfNotOpened() { 79 | inputStreamExtractor.next(); 80 | } 81 | 82 | @Test(expected = RuntimeException.class) 83 | public void openThrowsRuntimeExceptionIfRuntimeExceptionThrownByInputStreamMapper() { 84 | when(mockInputStreamMapper.apply(mockInputStream)).thenThrow(new RuntimeException("Bad AWS exception")); 85 | inputStreamExtractor.open(null); 86 | } 87 | 88 | @Test 89 | public void closeClosesInputStream() throws Exception { 90 | inputStreamExtractor.open(null); 91 | inputStreamExtractor.close(); 92 | verify(mockInputStream).close(); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/extractor/IterableExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.amazon.pocketEtl.exception.UnrecoverableStreamFailureException; 21 | 22 | import com.google.common.collect.ImmutableList; 23 | import org.junit.Test; 24 | 25 | import java.util.Iterator; 26 | import java.util.Optional; 27 | 28 | import static org.hamcrest.Matchers.equalTo; 29 | import static org.junit.Assert.assertThat; 30 | import static org.mockito.Mockito.mock; 31 | import static org.mockito.Mockito.when; 32 | 33 | @SuppressWarnings("unchecked") 34 | public class IterableExtractorTest extends EtlTestBase { 35 | @Test 36 | public void iteratorExtractorExtractsAndIterates() throws Exception { 37 | try (Extractor extractor = IterableExtractor.of(ImmutableList.of("1", "2", "3"))) { 38 | extractor.open(mockMetrics); 39 | assertThat(extractor.next(), equalTo(Optional.of("1"))); 40 | assertThat(extractor.next(), equalTo(Optional.of("2"))); 41 | assertThat(extractor.next(), equalTo(Optional.of("3"))); 42 | assertThat(extractor.next(), equalTo(Optional.empty())); 43 | } 44 | } 45 | 46 | @Test(expected = IllegalStateException.class) 47 | public void callingNextBeforeOpenThrowsIllegalStateException() { 48 | IterableExtractor.of(ImmutableList.of("1", "2", "3")).next(); 49 | } 50 | 51 | @Test(expected = UnrecoverableStreamFailureException.class) 52 | public void nextThrowsRuntimeExceptionThrowsAsUnrecoverableStreamFailureException() { 53 | Iterable mockIterable = mock(Iterable.class); 54 | Iterator mockIterator = mock(Iterator.class); 55 | when(mockIterable.iterator()).thenReturn(mockIterator); 56 | when(mockIterator.hasNext()).thenReturn(true); 57 | when(mockIterator.next()).thenThrow(new RuntimeException("Test exception")); 58 | 59 | Extractor iterableExtractor = IterableExtractor.of(mockIterable); 60 | iterableExtractor.open(null); 61 | iterableExtractor.next(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/extractor/IteratorExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.amazon.pocketEtl.EtlTestBase; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.google.common.collect.ImmutableList; 21 | import org.junit.Test; 22 | 23 | import java.util.Optional; 24 | 25 | import static org.hamcrest.Matchers.equalTo; 26 | import static org.junit.Assert.assertThat; 27 | 28 | public class IteratorExtractorTest extends EtlTestBase { 29 | @Test 30 | public void iteratorExtractorExtractsAndIterates() throws Exception { 31 | try (Extractor extractor = IteratorExtractor.of(ImmutableList.of(1, 2, 3).iterator())) { 32 | extractor.open(mockMetrics); 33 | assertThat(extractor.next(), equalTo(Optional.of(1))); 34 | assertThat(extractor.next(), equalTo(Optional.of(2))); 35 | assertThat(extractor.next(), equalTo(Optional.of(3))); 36 | assertThat(extractor.next(), equalTo(Optional.empty())); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/extractor/S3BufferedExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import com.amazonaws.services.s3.AmazonS3; 19 | import com.amazonaws.services.s3.model.S3Object; 20 | import com.amazonaws.services.s3.model.S3ObjectInputStream; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.mockito.Mock; 26 | import org.mockito.junit.MockitoJUnitRunner; 27 | 28 | import java.util.Iterator; 29 | import java.util.Optional; 30 | 31 | import static org.hamcrest.Matchers.equalTo; 32 | import static org.junit.Assert.assertThat; 33 | import static org.mockito.ArgumentMatchers.any; 34 | import static org.mockito.ArgumentMatchers.anyString; 35 | import static org.mockito.Mockito.verify; 36 | import static org.mockito.Mockito.when; 37 | 38 | @RunWith(MockitoJUnitRunner.class) 39 | public class S3BufferedExtractorTest { 40 | private final static String S3_BUCKET = "s3-bucket"; 41 | private final static String S3_KEY = "s3-key"; 42 | 43 | @Mock 44 | private InputStreamMapper mockInputStreamMapper; 45 | @Mock 46 | private AmazonS3 mockAmazonS3; 47 | @Mock 48 | private S3Object mockS3Object; 49 | @Mock 50 | private S3ObjectInputStream mockS3ObjectInputStream; 51 | 52 | private final Iterator testIterator = ImmutableList.of("one", "two", "three").iterator(); 53 | 54 | private S3BufferedExtractor s3BufferedExtractor; 55 | 56 | @Before 57 | public void createS3BufferedExtractor() { 58 | s3BufferedExtractor = S3BufferedExtractor.supplierOf(S3_BUCKET, S3_KEY, mockInputStreamMapper) 59 | .withClient(mockAmazonS3) 60 | .get(); 61 | } 62 | 63 | @Before 64 | public void stubAmazonS3() throws Exception { 65 | when(mockAmazonS3.getObject(anyString(), anyString())).thenReturn(mockS3Object); 66 | when(mockS3Object.getObjectContent()).thenReturn(mockS3ObjectInputStream); 67 | when(mockS3ObjectInputStream.read(any())).thenReturn(-1); 68 | } 69 | 70 | @Before 71 | public void stubIterator() { 72 | when(mockInputStreamMapper.apply(any())).thenReturn(testIterator); 73 | } 74 | 75 | @Test 76 | public void supplierConstructsAnInputStreamExtractorThatReadsFromS3() throws Exception { 77 | s3BufferedExtractor.open(null); 78 | s3BufferedExtractor.next(); 79 | s3BufferedExtractor.close(); 80 | 81 | verify(mockS3ObjectInputStream).read(any()); 82 | } 83 | 84 | @Test 85 | public void supplierConstructsAnInputStreamExtractorThatExtractsValues() throws Exception { 86 | s3BufferedExtractor.open(null); 87 | 88 | assertThat(s3BufferedExtractor.next(), equalTo(Optional.of("one"))); 89 | assertThat(s3BufferedExtractor.next(), equalTo(Optional.of("two"))); 90 | assertThat(s3BufferedExtractor.next(), equalTo(Optional.of("three"))); 91 | assertThat(s3BufferedExtractor.next(), equalTo(Optional.empty())); 92 | 93 | s3BufferedExtractor.close(); 94 | 95 | verify(mockS3ObjectInputStream).read(any()); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/extractor/TestDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.extractor; 17 | 18 | import org.joda.time.DateTime; 19 | 20 | import java.time.Instant; 21 | import java.util.List; 22 | import java.util.Objects; 23 | import java.util.Optional; 24 | 25 | public class TestDTO { 26 | private int aInt; 27 | private String aString; 28 | private double aDouble; 29 | private boolean aBoolean; 30 | private DateTime aDateTime; 31 | private Instant java8DateTime; 32 | private List aList; 33 | private Optional optionalString; 34 | 35 | public TestDTO() { 36 | // no-op 37 | } 38 | 39 | public TestDTO(int aInt, String aString, double aDouble, boolean aBoolean, DateTime aDateTime, Instant java8DateTime, 40 | List aList, Optional optionalString 41 | ) { 42 | this.aInt = aInt; 43 | this.aString = aString; 44 | this.aDouble = aDouble; 45 | this.aBoolean = aBoolean; 46 | this.aDateTime = aDateTime; 47 | this.java8DateTime = java8DateTime; 48 | this.aList = aList; 49 | this.optionalString = optionalString; 50 | } 51 | 52 | public int getAInt() { 53 | return aInt; 54 | } 55 | 56 | public void setAInt(int aInt) { 57 | this.aInt = aInt; 58 | } 59 | 60 | public String getAString() { 61 | return aString; 62 | } 63 | 64 | public void setAString(String aString) { 65 | this.aString = aString; 66 | } 67 | 68 | public double getADouble() { 69 | return aDouble; 70 | } 71 | 72 | public void setADouble(double aDouble) { 73 | this.aDouble = aDouble; 74 | } 75 | 76 | public boolean isABoolean() { 77 | return aBoolean; 78 | } 79 | 80 | public void setABoolean(boolean aBoolean) { 81 | this.aBoolean = aBoolean; 82 | } 83 | 84 | public DateTime getADateTime() { 85 | return aDateTime; 86 | } 87 | 88 | public void setADateTime(DateTime aDateTime) { 89 | this.aDateTime = aDateTime; 90 | } 91 | 92 | public Instant getJava8DateTime() { 93 | return java8DateTime; 94 | } 95 | 96 | public void setJava8DateTime(Instant java8DateTime) { 97 | this.java8DateTime = java8DateTime; 98 | } 99 | 100 | public List getAList() { 101 | return aList; 102 | } 103 | 104 | public void setAList(List aList) { 105 | this.aList = aList; 106 | } 107 | 108 | public Optional getOptionalString() { 109 | return optionalString; 110 | } 111 | 112 | public void setOptionalString(Optional optionalString) { 113 | this.optionalString = optionalString; 114 | } 115 | 116 | @Override 117 | public boolean equals(Object o) { 118 | if (this == o) return true; 119 | if (o == null || getClass() != o.getClass()) return false; 120 | TestDTO testDTO = (TestDTO) o; 121 | return aInt == testDTO.aInt && 122 | Double.compare(testDTO.aDouble, aDouble) == 0 && 123 | aBoolean == testDTO.aBoolean && 124 | Objects.equals(aString, testDTO.aString) && 125 | Objects.equals(aDateTime, testDTO.aDateTime) && 126 | Objects.equals(java8DateTime, testDTO.java8DateTime) && 127 | Objects.equals(aList, testDTO.aList) && 128 | Objects.equals(optionalString, testDTO.optionalString); 129 | } 130 | 131 | @Override 132 | public String toString() { 133 | return "TestDTO{" + 134 | "aInt=" + aInt + 135 | ", aString='" + aString + '\'' + 136 | ", aDouble=" + aDouble + 137 | ", aBoolean=" + aBoolean + 138 | ", aDateTime=" + aDateTime + 139 | ", java8DateTime=" + java8DateTime + 140 | ", aList=" + aList + 141 | ", optionalString=" + optionalString + 142 | '}'; 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/integration/db/SqlStringHelpersTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.integration.db; 17 | 18 | import com.google.common.collect.ImmutableList; 19 | import com.google.common.collect.ImmutableMap; 20 | import org.junit.Test; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | import static org.hamcrest.MatcherAssert.assertThat; 27 | import static org.hamcrest.Matchers.is; 28 | 29 | public class SqlStringHelpersTest { 30 | 31 | private static final String INPUT_SQL = "SELECT * FROM User WHERE firstName = ${firstName} AND lastName = ${lastName}"; 32 | private static final Map FIRST_NAME_SUBSTITUTION = ImmutableMap.of("firstName", "'Billy'"); 33 | private static final Map LAST_NAME_SUBSTITUTION = ImmutableMap.of("lastName", "'The Kid'"); 34 | private static final Map OCCUPATION_SUBSTITUTION = ImmutableMap.of("occupation", "'outlaw'"); 35 | 36 | @Test 37 | public void listToSqlStringWithPopulatedListReturnsCommaSeparatedStrings() { 38 | List listOfStrings = ImmutableList.of("a", "b", "c"); 39 | String expected = "'a', 'b', 'c'"; 40 | String actual = SqlStringHelpers.listToSQLString(listOfStrings); 41 | assertThat(actual, is(expected)); 42 | } 43 | 44 | @Test 45 | public void listToSqlStringWithSingleElementInListReturnsSingleQuotesNoCommas() { 46 | List singleStringList = ImmutableList.of("a"); 47 | String expected = "'a'"; 48 | String actual = SqlStringHelpers.listToSQLString(singleStringList); 49 | assertThat(actual, is(expected)); 50 | } 51 | 52 | @Test 53 | public void listToSqlStringReturnsEmptyStringIfPassedNull() { 54 | String expected = ""; 55 | String actual = SqlStringHelpers.listToSQLString(null); 56 | assertThat(actual, is(expected)); 57 | } 58 | 59 | @Test 60 | public void listToSqlStringReturnsEmptyStringIfPassedEmptyList() { 61 | String expected = ""; 62 | String actual = SqlStringHelpers.listToSQLString(new ArrayList<>()); 63 | assertThat(actual, is(expected)); 64 | } 65 | 66 | @Test 67 | public void substituteSQLForSingleSubstitutionWillReplaceKeyWithValue() { 68 | String expectedSQL = "SELECT * FROM User WHERE firstName = 'Billy' AND lastName = ${lastName}"; 69 | 70 | String result = SqlStringHelpers.substituteSQL(INPUT_SQL, FIRST_NAME_SUBSTITUTION); 71 | 72 | assertThat(result, is(expectedSQL)); 73 | } 74 | 75 | @Test 76 | public void substituteSQLForMultipleSubstitutionsWillReplaceKeyWithValue() { 77 | 78 | Map substitutions = ImmutableMap.builder().putAll(FIRST_NAME_SUBSTITUTION) 79 | .putAll(LAST_NAME_SUBSTITUTION).build(); 80 | String expectedSQL = "SELECT * FROM User WHERE firstName = 'Billy' AND lastName = 'The Kid'"; 81 | 82 | String result = SqlStringHelpers.substituteSQL(INPUT_SQL, substitutions); 83 | 84 | assertThat(result, is(expectedSQL)); 85 | } 86 | 87 | @Test 88 | public void substituteSQLReturnsUnmodifiedSQLGivenEmptyList() { 89 | String result = SqlStringHelpers.substituteSQL(INPUT_SQL, ImmutableMap.of()); 90 | assertThat(result, is(INPUT_SQL)); 91 | } 92 | 93 | @Test 94 | public void substituteSQLReturnsUnmodifiedSQLGivenNullList() { 95 | String result = SqlStringHelpers.substituteSQL(INPUT_SQL, null); 96 | assertThat(result, is(INPUT_SQL)); 97 | } 98 | 99 | @Test 100 | public void substituteSQLDoesNotModifyKeyIfSubstitutionKeyDoesNotFindMatch() { 101 | Map substitutions = ImmutableMap.builder().putAll(FIRST_NAME_SUBSTITUTION) 102 | .putAll(OCCUPATION_SUBSTITUTION).build(); 103 | 104 | String expectedSQL = "SELECT * FROM User WHERE firstName = 'Billy' AND lastName = ${lastName}"; 105 | String result = SqlStringHelpers.substituteSQL(INPUT_SQL, substitutions); 106 | assertThat(result, is(expectedSQL)); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/transformer/FilterTransformerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer; 17 | 18 | import com.amazon.pocketEtl.lookup.Lookup; 19 | import com.google.common.collect.ImmutableList; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.mockito.Mock; 24 | import org.mockito.junit.MockitoJUnitRunner; 25 | 26 | import java.util.List; 27 | import java.util.function.BiPredicate; 28 | 29 | import static org.hamcrest.Matchers.equalTo; 30 | import static org.junit.Assert.assertThat; 31 | import static org.mockito.ArgumentMatchers.anyString; 32 | import static org.mockito.ArgumentMatchers.eq; 33 | import static org.mockito.Mockito.when; 34 | 35 | @RunWith(MockitoJUnitRunner.class) 36 | public class FilterTransformerTest { 37 | 38 | private static final String SAMPLE_STRING_ONE = "sampleStringOne"; 39 | private static final String SAMPLE_STRING_TWO = "sampleStringTwo"; 40 | 41 | @Mock 42 | private BiPredicate> mockFilter; 43 | 44 | @Mock 45 | private Lookup mockLookup; 46 | 47 | private FilterTransformer filterTransformer; 48 | 49 | @Before 50 | public void stubFilterAndLookup() { 51 | when(mockFilter.test(anyString(), eq(mockLookup))).thenReturn(false); 52 | when(mockFilter.test(SAMPLE_STRING_ONE, mockLookup)).thenReturn(true); 53 | } 54 | 55 | @Before 56 | public void initializeFilterTransformer() { 57 | filterTransformer = new FilterTransformer<>(mockFilter, mockLookup); 58 | } 59 | 60 | @Test 61 | public void transformReturnsListOfObject() throws Exception { 62 | List transformedObjects = filterTransformer.transform(SAMPLE_STRING_ONE); 63 | assertThat(transformedObjects, equalTo(ImmutableList.of(SAMPLE_STRING_ONE))); 64 | } 65 | 66 | @Test 67 | public void transformReturnsEmptyList() throws Exception { 68 | List transformedObjects = filterTransformer.transform(SAMPLE_STRING_TWO); 69 | assertThat(transformedObjects, equalTo(ImmutableList.of())); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/transformer/MapTransformerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer; 17 | 18 | import com.google.common.collect.ImmutableList; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.mockito.junit.MockitoJUnitRunner; 22 | 23 | import java.util.List; 24 | 25 | import static org.hamcrest.Matchers.equalTo; 26 | import static org.junit.Assert.assertThat; 27 | 28 | @RunWith(MockitoJUnitRunner.class) 29 | public class MapTransformerTest { 30 | private MapTransformer mapTransformer = MapTransformer.of(String::toLowerCase); 31 | 32 | @Test 33 | public void transformCanTransformAnObject() { 34 | List result = mapTransformer.transform("TEST"); 35 | 36 | assertThat(result, equalTo(ImmutableList.of("test"))); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/pocketEtl/transformer/filter/ContainsFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.pocketEtl.transformer.filter; 17 | 18 | import com.amazon.pocketEtl.lookup.Lookup; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.mockito.Mock; 23 | import org.mockito.junit.MockitoJUnitRunner; 24 | 25 | import java.util.Optional; 26 | 27 | import static org.hamcrest.core.Is.is; 28 | import static org.junit.Assert.assertThat; 29 | import static org.mockito.ArgumentMatchers.anyString; 30 | import static org.mockito.ArgumentMatchers.eq; 31 | import static org.mockito.Mockito.when; 32 | 33 | @RunWith(MockitoJUnitRunner.class) 34 | public class ContainsFilterTest { 35 | private final static String TEST_STRING_ONE = "TestStringOne"; 36 | private final static String TEST_STRING_TWO = "TestStringTwo"; 37 | 38 | @Mock 39 | private Lookup mockLookup; 40 | 41 | private ContainsFilter containsFilter = new ContainsFilter<>(); 42 | 43 | @Before 44 | public void stubMockLookup() { 45 | when(mockLookup.get(anyString())).thenReturn(Optional.empty()); 46 | when(mockLookup.get(eq(TEST_STRING_ONE))).thenReturn(Optional.of(TEST_STRING_ONE)); 47 | } 48 | 49 | @Test 50 | public void testReturnsTrueIfObjectIsFoundInLookup() { 51 | boolean result = containsFilter.test(TEST_STRING_ONE, mockLookup); 52 | 53 | assertThat(result, is(true)); 54 | } 55 | 56 | @Test 57 | public void testReturnsFalseIfObjectIsNotFoundInLookup() { 58 | boolean result = containsFilter.test(TEST_STRING_TWO, mockLookup); 59 | 60 | assertThat(result, is(false)); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/BufferExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.Extractor; 20 | 21 | import java.util.Iterator; 22 | import java.util.List; 23 | import java.util.Optional; 24 | 25 | class BufferExtractor implements Extractor { 26 | 27 | private final Iterator bufferIterator; 28 | 29 | BufferExtractor(List buffer) { 30 | bufferIterator = buffer.iterator(); 31 | } 32 | 33 | @Override 34 | public Optional next() { 35 | if(bufferIterator.hasNext()) { 36 | return Optional.of(bufferIterator.next()); 37 | } 38 | 39 | return Optional.empty(); 40 | } 41 | 42 | @Override 43 | public void close() { 44 | //no-op 45 | } 46 | 47 | @Override 48 | public void open(EtlMetrics parentMetrics) { 49 | //no-op 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/BufferLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.Loader; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | import java.util.concurrent.ConcurrentLinkedQueue; 24 | 25 | public class BufferLoader implements Loader { 26 | private final ConcurrentLinkedQueue bufferLinkedQueue = new ConcurrentLinkedQueue<>(); 27 | 28 | @Override 29 | public void load(T objectToLoad) { 30 | bufferLinkedQueue.add(objectToLoad); 31 | } 32 | 33 | @Override 34 | public void open(EtlMetrics parentMetrics) { 35 | //no-op 36 | } 37 | 38 | @Override 39 | public void close() throws Exception { 40 | //no-op 41 | } 42 | 43 | List getBuffer() { 44 | return new ArrayList<>(bufferLinkedQueue); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/DynamoDbFunctionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.loader.DynamoDbLoader; 19 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 20 | import com.amazonaws.services.dynamodbv2.local.embedded.DynamoDBEmbedded; 21 | import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; 22 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 23 | import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; 24 | import com.amazonaws.services.dynamodbv2.model.GetItemRequest; 25 | import com.amazonaws.services.dynamodbv2.model.GetItemResult; 26 | import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; 27 | import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; 28 | import org.junit.Before; 29 | import org.junit.Test; 30 | 31 | import java.util.HashMap; 32 | 33 | import static org.hamcrest.CoreMatchers.is; 34 | import static org.hamcrest.CoreMatchers.nullValue; 35 | import static org.hamcrest.MatcherAssert.assertThat; 36 | 37 | public class DynamoDbFunctionalTest { 38 | private AmazonDynamoDB ddb; 39 | 40 | private String tableName; 41 | 42 | @Before 43 | public void setup() { 44 | tableName = "etl001"; 45 | 46 | ddb = DynamoDBEmbedded.create().amazonDynamoDB(); 47 | 48 | CreateTableRequest createTableRequest = new CreateTableRequest() 49 | .withTableName(tableName) 50 | .withProvisionedThroughput(new ProvisionedThroughput(10L, 10L)) 51 | .withAttributeDefinitions(new AttributeDefinition("pk", "S")) 52 | .withKeySchema(new KeySchemaElement("pk", "HASH")); 53 | 54 | ddb.createTable(createTableRequest); 55 | } 56 | 57 | @Test 58 | public void testHappyCaseLoading() { 59 | final DynamoDbLoader loader = DynamoDbLoader.of(tableName, "pk", Thing::getSomeUniqueId).withClient(ddb); 60 | 61 | // add an item 62 | { 63 | assertThat(getThingFromDdb("monolith").getItem(), is(nullValue())); 64 | 65 | Thing t001 = new Thing(); 66 | t001.someUniqueId = "monolith"; 67 | t001.year = "2001"; 68 | loader.load(t001); 69 | 70 | assertThat(getThingFromDdb("monolith").getItem().get("document").getM().get("year").getS(), is("2001")); 71 | } 72 | 73 | // add another item 74 | { 75 | assertThat(getThingFromDdb("macrolith").getItem(), is(nullValue())); 76 | 77 | Thing t001 = new Thing(); 78 | t001.someUniqueId = "macrolith"; 79 | t001.year = "2525"; 80 | loader.load(t001); 81 | 82 | assertThat(getThingFromDdb("monolith").getItem().get("document").getM().get("year").getS(), is("2001")); 83 | assertThat(getThingFromDdb("macrolith").getItem().get("document").getM().get("year").getS(), is("2525")); 84 | } 85 | } 86 | 87 | private GetItemResult getThingFromDdb(String key) { 88 | final HashMap requestItems = new HashMap<>(); 89 | requestItems.put("pk", new AttributeValue(key)); 90 | final GetItemRequest getItemRequest = new GetItemRequest(); 91 | getItemRequest.withTableName(tableName).withKey(requestItems); 92 | return ddb.getItem(getItemRequest); 93 | } 94 | 95 | static class Thing { 96 | public String someUniqueId; 97 | public String year; 98 | 99 | public String getSomeUniqueId() { 100 | return someUniqueId; 101 | } 102 | 103 | public String getYear() { 104 | return year; 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/FilterFunctionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.amazon.pocketEtl.Transformer; 21 | import com.amazon.pocketEtl.lookup.CachingLoaderLookup; 22 | import com.amazon.pocketEtl.lookup.Lookup; 23 | import com.amazon.pocketEtl.transformer.FilterTransformer; 24 | import com.amazon.pocketEtl.transformer.filter.ContainsFilter; 25 | import com.google.common.collect.ImmutableList; 26 | import org.junit.Test; 27 | 28 | import java.util.List; 29 | import java.util.concurrent.Semaphore; 30 | import java.util.function.BiPredicate; 31 | 32 | import static com.amazon.pocketEtl.EtlConsumerStage.load; 33 | import static com.amazon.pocketEtl.EtlConsumerStage.transform; 34 | import static org.hamcrest.MatcherAssert.assertThat; 35 | import static org.hamcrest.Matchers.containsInAnyOrder; 36 | import static org.hamcrest.Matchers.is; 37 | 38 | public class FilterFunctionalTest { 39 | private final static List INPUT_LIST_1 = ImmutableList.of(TestDTO.ONE, TestDTO.TWO, TestDTO.THREE); 40 | private final static List INPUT_LIST_2 = ImmutableList.of(TestDTO.FOUR, TestDTO.FIVE, TestDTO.SIX); 41 | private final static List INPUT_LIST_3 = ImmutableList.of(TestDTO.SEVEN, TestDTO.EIGHT, TestDTO.NINE); 42 | 43 | private final static List FILTER_LIST_1 = ImmutableList.of(TestDTO.ONE, TestDTO.FIVE, TestDTO.TEN); 44 | private final static List FILTER_LIST_2 = ImmutableList.of(TestDTO.ELEVEN, TestDTO.NINE, TestDTO.ZERO); 45 | 46 | private final static TestDTO[] EXPECTED_RESULT = {TestDTO.ONE, TestDTO.FIVE, TestDTO.NINE}; 47 | private final static TestDTO[] EXPECTED_NEGATIVE_RESULT = {TestDTO.TWO, TestDTO.THREE, TestDTO.FOUR, TestDTO.SIX, TestDTO.SEVEN, TestDTO.EIGHT}; 48 | 49 | private BufferLoader resultBufferLoader = new BufferLoader<>(); 50 | private ThrowsEtlConsumer errorConsumer = new ThrowsEtlConsumer(); 51 | 52 | private void runFilterTest(BiPredicate> filter) throws Exception { 53 | CachingLoaderLookup cachingLoaderLookup = new CachingLoaderLookup<>(Semaphore::new); 54 | Transformer filterTransformer = new FilterTransformer<>(filter, cachingLoaderLookup); 55 | 56 | Extractor[] extractors = { 57 | new BufferExtractor<>(INPUT_LIST_1), 58 | new BufferExtractor<>(INPUT_LIST_2), 59 | new BufferExtractor<>(INPUT_LIST_3) 60 | }; 61 | 62 | EtlStream cacheLoaderStream = 63 | EtlStream.extract(new BufferExtractor<>(FILTER_LIST_1), new BufferExtractor<>(FILTER_LIST_2)) 64 | .then(load(TestDTO.class, cachingLoaderLookup).withThreads(5)); 65 | 66 | EtlStream sourceEtlStream = EtlStream.extract(extractors) 67 | .then(transform(TestDTO.class, filterTransformer).withThreads(5)); 68 | 69 | EtlStream.combine(sourceEtlStream, cacheLoaderStream) 70 | .load(TestDTO.class, resultBufferLoader) 71 | .run(); 72 | 73 | assertThat(errorConsumer.isExceptionWasThrown(), is(false)); 74 | } 75 | 76 | @Test 77 | public void testContainsFilter() throws Exception { 78 | runFilterTest(new ContainsFilter<>()); 79 | 80 | assertThat(resultBufferLoader.getBuffer(), containsInAnyOrder(EXPECTED_RESULT)); 81 | } 82 | 83 | @Test 84 | public void testNotContainsFilter() throws Exception { 85 | runFilterTest(new ContainsFilter().negate()); 86 | 87 | assertThat(resultBufferLoader.getBuffer(), containsInAnyOrder(EXPECTED_NEGATIVE_RESULT)); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/ImmutabilityTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.extractor.IterableExtractor; 20 | import com.amazon.pocketEtl.transformer.MapTransformer; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | import static org.hamcrest.Matchers.equalTo; 28 | import static org.junit.Assert.assertThat; 29 | 30 | public class ImmutabilityTest { 31 | @Test 32 | public void testStreamImmutability() throws Exception { 33 | List inputData = ImmutableList.of(new TestDTO("ONE"), new TestDTO("TWO"), new TestDTO("THREE")); 34 | List outputData = new ArrayList<>(); 35 | EtlStream baseStream = EtlStream.extract(IterableExtractor.of(inputData)); 36 | 37 | baseStream.transform(TestDTO.class, MapTransformer.of(obj -> new TestDTO(obj.getValue().toLowerCase()))); 38 | baseStream.load(TestDTO.class, outputData::add) 39 | .run(); 40 | 41 | assertThat(outputData, equalTo(inputData)); 42 | } 43 | 44 | @Test 45 | public void testStreamReuse() throws Exception { 46 | List inputData = ImmutableList.of(new TestDTO("ONE"), new TestDTO("TWO")); 47 | List outputData = new ArrayList<>(); 48 | EtlStream baseStream = EtlStream.extract(IterableExtractor.of(inputData)); 49 | 50 | baseStream.transform(TestDTO.class, MapTransformer.of(obj -> new TestDTO(obj.getValue().toLowerCase()))) 51 | .load(TestDTO.class, outputData::add) 52 | .run(); 53 | 54 | baseStream.load(TestDTO.class, outputData::add) 55 | .run(); 56 | 57 | assertThat(outputData, equalTo(ImmutableList.of(new TestDTO("one"), new TestDTO("two"), new TestDTO("ONE"), 58 | new TestDTO("TWO")))); 59 | } 60 | 61 | @Test 62 | public void testCombineImmutability() throws Exception { 63 | List inputData1 = ImmutableList.of(new TestDTO("ONE")); 64 | List inputData2 = ImmutableList.of(new TestDTO("TWO")); 65 | List outputData = new ArrayList<>(); 66 | 67 | EtlStream baseStream1 = EtlStream.extract(IterableExtractor.of(inputData1)); 68 | EtlStream baseStream2 = EtlStream.extract(IterableExtractor.of(inputData2)); 69 | 70 | EtlStream combinedStream = EtlStream.combine(baseStream1, baseStream2); 71 | 72 | baseStream1.transform(TestDTO.class, MapTransformer.of(obj -> new TestDTO(obj.getValue().toLowerCase()))) 73 | .load(TestDTO.class, outputData::add) 74 | .run(); 75 | 76 | combinedStream.load(TestDTO.class, outputData::add) 77 | .run(); 78 | 79 | assertThat(outputData, equalTo(ImmutableList.of(new TestDTO("one"), new TestDTO("ONE"), new TestDTO("TWO")))); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/MockedS3FunctionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.Loader; 20 | import com.amazon.pocketEtl.extractor.IterableExtractor; 21 | import com.amazon.pocketEtl.loader.CsvStringSerializer; 22 | import com.amazon.pocketEtl.loader.S3FastLoader; 23 | import com.amazonaws.services.s3.AmazonS3; 24 | import com.amazonaws.services.s3.model.PutObjectRequest; 25 | import com.amazonaws.services.s3.model.PutObjectResult; 26 | import com.amazonaws.util.IOUtils; 27 | import com.google.common.collect.ImmutableList; 28 | import org.junit.Before; 29 | import org.junit.Test; 30 | import org.junit.runner.RunWith; 31 | import org.mockito.Mock; 32 | import org.mockito.junit.MockitoJUnitRunner; 33 | import org.mockito.stubbing.Answer; 34 | 35 | import java.io.InputStream; 36 | import java.util.ArrayList; 37 | import java.util.List; 38 | 39 | import static org.hamcrest.Matchers.equalTo; 40 | import static org.junit.Assert.assertThat; 41 | import static org.mockito.ArgumentMatchers.any; 42 | import static org.mockito.Mockito.when; 43 | 44 | @RunWith(MockitoJUnitRunner.class) 45 | public class MockedS3FunctionalTest { 46 | private final List outputStrings = new ArrayList<>(); 47 | 48 | @Mock 49 | private AmazonS3 mockAmazonS3; 50 | 51 | @Before 52 | public void prepareMockS3() { 53 | when(mockAmazonS3.putObject(any(PutObjectRequest.class))).thenAnswer((Answer) invocation -> { 54 | PutObjectRequest request = (PutObjectRequest)invocation.getArguments()[0]; 55 | InputStream inputStream = request.getInputStream(); 56 | outputStrings.add(IOUtils.toString(inputStream)); 57 | return null; 58 | }); 59 | } 60 | 61 | @Test 62 | public void testPartFilesWithHeadersOneRowPerFile() throws Exception { 63 | List inputData = ImmutableList.of(new TestDTO("ONE"), new TestDTO("TWO"), new TestDTO("THREE")); 64 | Loader loader = S3FastLoader.supplierOf("test-bucket", () -> CsvStringSerializer.of(TestDTO.class).withHeaderRow(true)) 65 | .withMaxPartFileSizeInBytes(10) 66 | .withClient(mockAmazonS3) 67 | .get(); 68 | 69 | EtlStream.extract(IterableExtractor.of(inputData)) 70 | .load(TestDTO.class, loader) 71 | .run(); 72 | 73 | List expectedOutput = ImmutableList.of("value\nONE\n", "value\nTWO\n", "value\nTHREE\n"); 74 | 75 | assertThat(outputStrings, equalTo(expectedOutput)); 76 | } 77 | 78 | @Test 79 | public void testPartFilesWithHeadersMultiRowsPerFile() throws Exception { 80 | List inputData = ImmutableList.of(new TestDTO("ONE"), new TestDTO("TWO"), new TestDTO("THREE"), 81 | new TestDTO("FOUR")); 82 | Loader loader = S3FastLoader.supplierOf("test-bucket", () -> CsvStringSerializer.of(TestDTO.class).withHeaderRow(true)) 83 | .withMaxPartFileSizeInBytes(17) 84 | .withClient(mockAmazonS3) 85 | .get(); 86 | 87 | EtlStream.extract(IterableExtractor.of(inputData)) 88 | .load(TestDTO.class, loader) 89 | .run(); 90 | 91 | List expectedOutput = ImmutableList.of("value\nONE\nTWO\n", "value\nTHREE\nFOUR\n"); 92 | 93 | assertThat(outputStrings, equalTo(expectedOutput)); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/SimpleEtlFunctionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.google.common.collect.ImmutableList; 21 | import org.junit.Test; 22 | 23 | import java.util.List; 24 | 25 | import static org.hamcrest.MatcherAssert.assertThat; 26 | import static org.hamcrest.Matchers.containsInAnyOrder; 27 | import static org.hamcrest.Matchers.is; 28 | 29 | public class SimpleEtlFunctionalTest { 30 | private final static List INPUT_LIST_1 = ImmutableList.of(TestDTO.ONE, TestDTO.TWO, TestDTO.THREE); 31 | private final static List INPUT_LIST_2 = ImmutableList.of(TestDTO.FOUR, TestDTO.FIVE, TestDTO.SIX); 32 | private final static List INPUT_LIST_3 = ImmutableList.of(TestDTO.SEVEN, TestDTO.EIGHT, TestDTO.NINE); 33 | 34 | private final static TestDTO[] EXPECTED_RESULT = {TestDTO.ONE, TestDTO.TWO, TestDTO.THREE, TestDTO.FOUR, TestDTO.FIVE, 35 | TestDTO.SIX, TestDTO.SEVEN, TestDTO.EIGHT, TestDTO.NINE}; 36 | 37 | 38 | private BufferLoader resultBufferLoader = new BufferLoader<>(); 39 | private ThrowsEtlConsumer errorConsumer = new ThrowsEtlConsumer(); 40 | 41 | private void runSimpleTest() throws Exception { 42 | 43 | Extractor[] extractors = { 44 | new BufferExtractor<>(INPUT_LIST_1), 45 | new BufferExtractor<>(INPUT_LIST_2), 46 | new BufferExtractor<>(INPUT_LIST_3) 47 | }; 48 | 49 | EtlStream.extract(extractors) 50 | .load(TestDTO.class, resultBufferLoader) 51 | .run(); 52 | 53 | assertThat(errorConsumer.isExceptionWasThrown(), is(false)); 54 | } 55 | 56 | @Test 57 | public void testSimpleEtl() throws Exception { 58 | runSimpleTest(); 59 | 60 | assertThat(resultBufferLoader.getBuffer(), containsInAnyOrder(EXPECTED_RESULT)); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/SimpleFluentFunctionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.extractor.IterableExtractor; 20 | import com.amazon.pocketEtl.transformer.MapTransformer; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | import static com.amazon.pocketEtl.EtlConsumerStage.transform; 28 | import static org.hamcrest.Matchers.containsInAnyOrder; 29 | import static org.junit.Assert.assertThat; 30 | 31 | public class SimpleFluentFunctionalTest { 32 | @Test 33 | public void testSimpleFluentInterface() throws Exception { 34 | List inputData = ImmutableList.of(new TestDTO("ONE"), new TestDTO("TWO"), new TestDTO("THREE")); 35 | List outputData = new ArrayList<>(); 36 | 37 | EtlStream.extract(IterableExtractor.of(inputData)) 38 | .then(transform(TestDTO.class, 39 | MapTransformer.of(obj -> new TestDTO(obj.getValue().toLowerCase()))).withThreads(5)) 40 | .load(TestDTO.class, outputData::add) 41 | .run(); 42 | 43 | assertThat(outputData, containsInAnyOrder(new TestDTO("one"), new TestDTO("two"), new TestDTO("three"))); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/TestDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Data; 20 | import lombok.NoArgsConstructor; 21 | 22 | import javax.annotation.Nonnull; 23 | 24 | @Data 25 | @AllArgsConstructor 26 | @NoArgsConstructor 27 | class TestDTO implements Comparable { 28 | private String value; 29 | 30 | @Override 31 | public int compareTo(@Nonnull TestDTO testDTO) { 32 | return value.compareTo(testDTO.value); 33 | } 34 | 35 | static final TestDTO ZERO = new TestDTO("ZERO"); 36 | static final TestDTO ONE = new TestDTO("ONE"); 37 | static final TestDTO TWO = new TestDTO("TWO"); 38 | static final TestDTO THREE = new TestDTO("THREE"); 39 | static final TestDTO FOUR = new TestDTO("FOUR"); 40 | static final TestDTO FIVE = new TestDTO("FIVE"); 41 | static final TestDTO SIX = new TestDTO("SIX"); 42 | static final TestDTO SEVEN = new TestDTO("SEVEN"); 43 | static final TestDTO EIGHT = new TestDTO("EIGHT"); 44 | static final TestDTO NINE = new TestDTO("NINE"); 45 | static final TestDTO TEN = new TestDTO("TEN"); 46 | static final TestDTO ELEVEN = new TestDTO("ELEVEN"); 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/TestDTO2.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Data; 20 | import lombok.NoArgsConstructor; 21 | import org.joda.time.DateTime; 22 | 23 | @Data 24 | @AllArgsConstructor 25 | @NoArgsConstructor 26 | public class TestDTO2 { 27 | private Integer id; 28 | private String aString; 29 | private Integer aNumber; 30 | private DateTime aDateTime; 31 | private Boolean aBoolean; 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/ThrowsEtlConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlMetrics; 19 | import com.amazon.pocketEtl.core.EtlStreamObject; 20 | import com.amazon.pocketEtl.core.consumer.EtlConsumer; 21 | import lombok.AccessLevel; 22 | import lombok.Getter; 23 | 24 | public class ThrowsEtlConsumer implements EtlConsumer { 25 | @Getter(AccessLevel.PUBLIC) 26 | private boolean exceptionWasThrown = false; 27 | 28 | @Override 29 | public void consume(EtlStreamObject objectToConsume) throws IllegalStateException { 30 | exceptionWasThrown = true; 31 | throw new RuntimeException("ThrowsConsumer consumed object: " + objectToConsume.toString()); 32 | } 33 | 34 | @Override 35 | public void open(EtlMetrics parentMetrics) { 36 | // no-op 37 | } 38 | 39 | @Override 40 | public void close() { 41 | // no-op 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/functionalTests/TransformerFanoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package functionalTests; 17 | 18 | import com.amazon.pocketEtl.EtlStream; 19 | import com.amazon.pocketEtl.Extractor; 20 | import com.amazon.pocketEtl.Transformer; 21 | import com.google.common.collect.ImmutableList; 22 | import org.junit.Test; 23 | 24 | import java.util.List; 25 | import java.util.Locale; 26 | 27 | import static com.amazon.pocketEtl.EtlConsumerStage.load; 28 | import static com.amazon.pocketEtl.EtlConsumerStage.transform; 29 | import static com.amazon.pocketEtl.EtlProducerStage.extract; 30 | import static org.hamcrest.MatcherAssert.assertThat; 31 | import static org.hamcrest.Matchers.containsInAnyOrder; 32 | import static org.hamcrest.Matchers.is; 33 | 34 | public class TransformerFanoutTest { 35 | private final static List INPUT_LIST_1 = ImmutableList.of(TestDTO.ONE, TestDTO.TWO, TestDTO.THREE); 36 | private final static List INPUT_LIST_2 = ImmutableList.of(TestDTO.FOUR, TestDTO.FIVE, TestDTO.SIX); 37 | private final static List INPUT_LIST_3 = ImmutableList.of(TestDTO.SEVEN, TestDTO.EIGHT, TestDTO.NINE); 38 | 39 | private final static TestDTO[] EXPECTED_RESULT = {TestDTO.ONE, TestDTO.TWO, TestDTO.THREE, TestDTO.FOUR, TestDTO.FIVE, 40 | TestDTO.SIX, TestDTO.SEVEN, TestDTO.EIGHT, TestDTO.NINE, new TestDTO("one"), new TestDTO("two"), new TestDTO("three"), 41 | new TestDTO("four"), new TestDTO("five"), new TestDTO("six"), new TestDTO("seven"), new TestDTO("eight"), 42 | new TestDTO("nine")}; 43 | 44 | private BufferLoader resultBufferLoader = new BufferLoader<>(); 45 | private ThrowsEtlConsumer errorConsumer = new ThrowsEtlConsumer(); 46 | 47 | private Transformer lowerCaseFanoutTransformer = 48 | word -> ImmutableList.of(word, new TestDTO(word.getValue().toLowerCase(Locale.ENGLISH))); 49 | 50 | private void runTest() throws Exception { 51 | Extractor[] extractors = { 52 | new BufferExtractor<>(INPUT_LIST_1), 53 | new BufferExtractor<>(INPUT_LIST_2), 54 | new BufferExtractor<>(INPUT_LIST_3) 55 | }; 56 | 57 | EtlStream.from(extract(extractors)) 58 | .then(transform(TestDTO.class, lowerCaseFanoutTransformer).withThreads(5)) 59 | .then(load(TestDTO.class, resultBufferLoader).withThreads(5)) 60 | .run(); 61 | 62 | assertThat(errorConsumer.isExceptionWasThrown(), is(false)); 63 | } 64 | 65 | @Test 66 | public void testFanoutTransformer() throws Exception { 67 | runTest(); 68 | 69 | assertThat(resultBufferLoader.getBuffer(), containsInAnyOrder(EXPECTED_RESULT)); 70 | } 71 | } 72 | --------------------------------------------------------------------------------