├── .editorconfig ├── .gitignore ├── README.md ├── pom.xml └── src ├── main ├── java │ └── example │ │ ├── Application.java │ │ ├── KinesisMessageProcessor.java │ │ └── kinesis │ │ ├── AWSConfig.java │ │ ├── KinesisListener.java │ │ ├── RecordProcessor.java │ │ └── RecordProcessorFactory.java └── resources │ └── application.properties └── test ├── java └── example │ └── kinesis │ ├── AWSConfig.java │ └── KinesisMessageProcessorIT.java └── resources └── application-integration-test.properties /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | indent_size = 2 11 | indent_style = space 12 | 13 | [Makefile] 14 | indent_style = tab 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | *.iml 4 | .DS_Store 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Kinesis with Localstack 2 | This is an application example that simulates AWS Kinesis using [Localstack](https://github.com/localstack/localstack) and docker containers -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | Kinesis with Localstack 7 | Application example that simulates AWS Kinesis using Localstack 8 | 9 | 10 | 11 | org.apache.maven.plugins 12 | maven-compiler-plugin 13 | 14 | 1.8 15 | 1.8 16 | 17 | 18 | 19 | 20 | 21 | example.kinesis-with-localstack 22 | 23 | 24 | org.springframework.boot 25 | spring-boot-starter-parent 26 | 1.5.9.RELEASE 27 | 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-web 33 | 34 | 35 | org.springframework.boot 36 | spring-boot-starter-test 37 | 38 | 39 | org.projectlombok 40 | lombok 41 | 1.16.18 42 | 43 | 44 | com.amazonaws 45 | amazon-kinesis-client 46 | 1.8.8 47 | 48 | 49 | cloud.localstack 50 | localstack-utils 51 | 0.1.13 52 | 53 | 54 | 55 | 56 | org.assertj 57 | assertj-core 58 | 3.9.0 59 | test 60 | 61 | 62 | org.awaitility 63 | awaitility 64 | 3.0.0 65 | test 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /src/main/java/example/Application.java: -------------------------------------------------------------------------------- 1 | package example; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | 7 | @Slf4j 8 | @SpringBootApplication 9 | public class Application { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(Application.class, args); 13 | log.info("Started Application ..."); 14 | } 15 | } 16 | 17 | -------------------------------------------------------------------------------- /src/main/java/example/KinesisMessageProcessor.java: -------------------------------------------------------------------------------- 1 | package example; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.stereotype.Component; 5 | 6 | @Slf4j 7 | @Component 8 | public class KinesisMessageProcessor { 9 | 10 | public void processKinesisMessage(String message) { 11 | log.info("Message received from the Kinesis stream: " + message); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/example/kinesis/AWSConfig.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import com.amazonaws.auth.AWSCredentialsProvider; 4 | import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; 5 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 6 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; 7 | import com.amazonaws.services.kinesis.AmazonKinesis; 8 | import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder; 9 | import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; 10 | import org.springframework.context.annotation.Profile; 11 | import org.springframework.stereotype.Component; 12 | 13 | @Component 14 | @Profile("default") 15 | public class AWSConfig { 16 | public AmazonKinesis kinesisClient() { 17 | return AmazonKinesisClientBuilder.standard().withCredentials(getCredentials()).build(); 18 | } 19 | 20 | public AmazonDynamoDB dynamoDBClient() { 21 | return AmazonDynamoDBClientBuilder.standard().withCredentials(getCredentials()).build(); 22 | } 23 | 24 | public AWSCredentialsProvider getCredentials() { 25 | return new DefaultAWSCredentialsProviderChain(); 26 | } 27 | 28 | public MetricsLevel getCloudWatchMetricsLevel() { 29 | return MetricsLevel.DETAILED; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/example/kinesis/KinesisListener.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; 4 | import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration; 5 | import com.amazonaws.services.kinesis.clientlibrary.lib.worker.Worker; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.beans.factory.annotation.Value; 9 | import org.springframework.context.SmartLifecycle; 10 | import org.springframework.stereotype.Component; 11 | 12 | import java.util.UUID; 13 | 14 | @Slf4j 15 | @Component 16 | public class KinesisListener implements SmartLifecycle { 17 | 18 | @Value("${kinesis.streamName}") 19 | private String kinesisStreamName; 20 | 21 | @Value("${application.name}") 22 | private String applicationName; 23 | 24 | private final AWSConfig awsConfig; 25 | 26 | private Worker worker; 27 | private Thread workerThread; 28 | 29 | private RecordProcessorFactory recordProcessorFactory; 30 | 31 | @Autowired 32 | public KinesisListener(RecordProcessorFactory recordProcessorFactory, AWSConfig awsConfig) { 33 | this.recordProcessorFactory = recordProcessorFactory; 34 | this.awsConfig = awsConfig; 35 | } 36 | 37 | @Override 38 | public void start() { 39 | log.info("Starting the worker"); 40 | String workerId = applicationName + UUID.randomUUID(); 41 | final KinesisClientLibConfiguration config = new KinesisClientLibConfiguration( 42 | applicationName, 43 | kinesisStreamName, 44 | awsConfig.getCredentials(), 45 | workerId) 46 | .withMetricsLevel(awsConfig.getCloudWatchMetricsLevel()) 47 | .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); 48 | 49 | worker = new Worker.Builder() 50 | .recordProcessorFactory(recordProcessorFactory) 51 | .config(config) 52 | .kinesisClient(awsConfig.kinesisClient()) 53 | .dynamoDBClient(awsConfig.dynamoDBClient()) 54 | .build(); 55 | 56 | workerThread = new Thread(worker, "kinesisListener"); 57 | workerThread.start(); 58 | } 59 | 60 | @Override 61 | public void stop() { 62 | log.info("Stopping the worker"); 63 | 64 | try { 65 | worker.createGracefulShutdownCallable().call(); 66 | } catch (Exception e) { 67 | log.error("Shutting down the Kinesis Worker failed!", e); 68 | } 69 | } 70 | 71 | @Override 72 | public boolean isRunning() { 73 | return null != workerThread && workerThread.isAlive(); 74 | } 75 | 76 | @Override 77 | public boolean isAutoStartup() { 78 | return true; 79 | } 80 | 81 | @Override 82 | public void stop(Runnable callback) { 83 | stop(); 84 | } 85 | 86 | @Override 87 | public int getPhase() { 88 | return Integer.MAX_VALUE; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/example/kinesis/RecordProcessor.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException; 4 | import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException; 5 | import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; 6 | import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; 7 | import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason; 8 | import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput; 9 | import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput; 10 | import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput; 11 | import com.amazonaws.services.kinesis.model.Record; 12 | import example.KinesisMessageProcessor; 13 | import lombok.extern.slf4j.Slf4j; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.stereotype.Component; 16 | 17 | import java.nio.charset.StandardCharsets; 18 | 19 | @Slf4j 20 | @Component 21 | public class RecordProcessor implements IRecordProcessor { 22 | 23 | private KinesisMessageProcessor kinesisMessageProcessor; 24 | 25 | @Autowired 26 | public RecordProcessor(KinesisMessageProcessor kinesisMessageProcessor) { 27 | this.kinesisMessageProcessor = kinesisMessageProcessor; 28 | } 29 | 30 | @Override 31 | public void initialize(InitializationInput initializationInput) { 32 | log.info("Initializing RecordProcessor"); 33 | } 34 | 35 | @Override 36 | public void processRecords(ProcessRecordsInput processRecordsInput) { 37 | log.info(String.format("Processing %d records from Kinesis", processRecordsInput.getRecords().size())); 38 | 39 | processRecordsInput.getRecords().forEach(this::handleSingleRecord); 40 | checkpoint(processRecordsInput.getCheckpointer()); 41 | } 42 | 43 | private void handleSingleRecord(Record record) { 44 | String message = StandardCharsets.UTF_8.decode(record.getData()).toString(); 45 | kinesisMessageProcessor.processKinesisMessage(message); 46 | } 47 | 48 | @Override 49 | public void shutdown(ShutdownInput shutdownInput) { 50 | if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) { 51 | checkpoint(shutdownInput.getCheckpointer()); 52 | } 53 | log.info("Shutting down RecordProcessor"); 54 | } 55 | 56 | private void checkpoint(IRecordProcessorCheckpointer checkpointer) { 57 | try { 58 | checkpointer.checkpoint(); 59 | } catch (InvalidStateException e) { 60 | log.error("Failed to checkpoint. KCL threw an InvalidStateException", e); 61 | } catch (ShutdownException e) { 62 | log.error("Failed to checkpoint. The RecordProcessor instance has been shutdown", e); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/example/kinesis/RecordProcessorFactory.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor; 4 | import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | @Component 9 | public class RecordProcessorFactory implements IRecordProcessorFactory { 10 | 11 | private RecordProcessor recordProcessor; 12 | 13 | @Autowired 14 | public RecordProcessorFactory(RecordProcessor recordProcessor) { 15 | this.recordProcessor = recordProcessor; 16 | } 17 | 18 | @Override 19 | public IRecordProcessor createProcessor() { 20 | return recordProcessor; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | kinesis.streamName=kinesisStream 2 | application.name=kinesis-with-localstack-example 3 | -------------------------------------------------------------------------------- /src/test/java/example/kinesis/AWSConfig.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import cloud.localstack.DockerTestUtils; 4 | import com.amazonaws.auth.AWSCredentialsProvider; 5 | import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; 6 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 7 | import com.amazonaws.services.kinesis.AmazonKinesis; 8 | import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.stereotype.Component; 11 | 12 | @Component 13 | @Profile("integration-test") 14 | public class AWSConfig { 15 | 16 | public AmazonKinesis kinesisClient() { 17 | return DockerTestUtils.getClientKinesis(); 18 | } 19 | 20 | public AmazonDynamoDB dynamoDBClient() { 21 | return DockerTestUtils.getClientDynamoDb(); 22 | } 23 | 24 | public AWSCredentialsProvider getCredentials() { 25 | return new DefaultAWSCredentialsProviderChain(); 26 | } 27 | 28 | public MetricsLevel getCloudWatchMetricsLevel() { 29 | return MetricsLevel.NONE; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/example/kinesis/KinesisMessageProcessorIT.java: -------------------------------------------------------------------------------- 1 | package example.kinesis; 2 | 3 | import cloud.localstack.TestUtils; 4 | import cloud.localstack.docker.LocalstackDockerTestRunner; 5 | import cloud.localstack.docker.annotation.EC2HostNameResolver; 6 | import cloud.localstack.docker.annotation.LocalstackDockerProperties; 7 | import com.amazonaws.services.kinesis.AmazonKinesis; 8 | import com.amazonaws.services.kinesis.model.PutRecordRequest; 9 | import example.KinesisMessageProcessor; 10 | import org.junit.*; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.beans.factory.annotation.Value; 14 | import org.springframework.boot.test.context.SpringBootTest; 15 | import org.springframework.boot.test.mock.mockito.MockBean; 16 | import org.springframework.test.context.ActiveProfiles; 17 | import org.springframework.test.context.junit4.rules.SpringClassRule; 18 | import org.springframework.test.context.junit4.rules.SpringMethodRule; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import static org.awaitility.Awaitility.await; 23 | import static org.mockito.Mockito.timeout; 24 | import static org.mockito.Mockito.verify; 25 | 26 | /** 27 | * Make sure that docker is running locally 28 | */ 29 | @RunWith(LocalstackDockerTestRunner.class) 30 | @LocalstackDockerProperties(services = {"dynamodb", "kinesis"}) 31 | @SpringBootTest 32 | @ActiveProfiles("integration-test") 33 | public class KinesisMessageProcessorIT { 34 | 35 | @ClassRule 36 | public static final SpringClassRule springClassRule = new SpringClassRule(); 37 | @Rule 38 | public final SpringMethodRule springMethodRule = new SpringMethodRule(); 39 | 40 | @Value("${kinesis.streamName}") 41 | public String streamName; 42 | 43 | @MockBean 44 | private KinesisMessageProcessor kinesisMessageProcessor; 45 | 46 | @Autowired 47 | private AWSConfig awsConfig; 48 | 49 | @Autowired 50 | private KinesisListener listener; 51 | 52 | private AmazonKinesis kinesisClient; 53 | 54 | static { 55 | TestUtils.setEnv("AWS_ACCESS_KEY_ID", "some_aws_access_key_id"); 56 | TestUtils.setEnv("AWS_SECRET_ACCESS_KEY", "some_aws_secret_access_key"); 57 | 58 | // https://github.com/mhart/kinesalite/blob/master/README.md#cbor-protocol-issues-with-the-java-sdk 59 | TestUtils.setEnv("AWS_CBOR_DISABLE", "1"); 60 | } 61 | 62 | @Before 63 | public void setup() { 64 | kinesisClient = awsConfig.kinesisClient(); 65 | } 66 | 67 | @Test 68 | public void shouldProcessKinesisMessage() { 69 | givenThereIsAKinesisStream: 70 | { 71 | kinesisClient.createStream(streamName, 1); 72 | await().until(() -> 73 | kinesisClient.describeStream(streamName).getStreamDescription().getStreamStatus().equals("ACTIVE") 74 | ); 75 | } 76 | 77 | whenThereIsARecordInTheStream: 78 | { 79 | PutRecordRequest putRecordRequest = new PutRecordRequest() 80 | .withStreamName(streamName) 81 | .withPartitionKey("some_partition_key") 82 | .withData(ByteBuffer.wrap("Hello".getBytes())); 83 | kinesisClient.putRecord(putRecordRequest); 84 | } 85 | 86 | thenTheReaderReadsTheMessageFromTheStream: 87 | { 88 | verify(kinesisMessageProcessor, timeout(30000)).processKinesisMessage("Hello"); 89 | } 90 | } 91 | 92 | @After 93 | public void shutDownWorkerAndDeleteSetup() { 94 | listener.stop(); 95 | kinesisClient.deleteStream(streamName); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/test/resources/application-integration-test.properties: -------------------------------------------------------------------------------- 1 | kinesis.streamName=kinesisStream 2 | application.name=kinesis-with-localstack-example 3 | --------------------------------------------------------------------------------