├── .gitignore ├── CODEOWNERS ├── nakadi-producer-starter-spring-boot-2-test ├── src │ ├── main │ │ ├── resources │ │ │ ├── db │ │ │ │ └── migration │ │ │ │ │ └── V1.1__noop.sql │ │ │ └── application.yml │ │ └── java │ │ │ └── org │ │ │ └── zalando │ │ │ └── nakadiproducer │ │ │ └── tests │ │ │ └── Application.java │ └── test │ │ ├── resources │ │ └── tokens │ │ │ ├── nakadi-token-type │ │ │ └── nakadi-token-secret │ │ └── java │ │ └── org │ │ └── zalando │ │ └── nakadiproducer │ │ └── tests │ │ ├── MockNakadiConfig.java │ │ └── ApplicationIT.java └── pom.xml ├── MAINTAINERS ├── nakadi-producer-loadtest ├── src │ ├── main │ │ ├── resources │ │ │ ├── db │ │ │ │ └── migration │ │ │ │ │ └── V1__create_nakadi_events_schema.sql │ │ │ └── application.yaml │ │ └── java │ │ │ └── org │ │ │ └── zalando │ │ │ └── nakadiproducer │ │ │ ├── event │ │ │ └── ExampleBusinessEvent.java │ │ │ └── Application.java │ └── test │ │ ├── java │ │ └── org │ │ │ └── zalando │ │ │ └── nakadiproducer │ │ │ ├── configuration │ │ │ ├── AopConfiguration.java │ │ │ └── TokenConfiguration.java │ │ │ ├── interceptor │ │ │ └── ProfilerInterceptor.java │ │ │ └── LoadTestIT.java │ │ └── resources │ │ ├── example-business-event.json │ │ ├── application.yaml │ │ └── docker-compose.yaml ├── README.md └── pom.xml ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── nakadi-producer-spring-boot-starter ├── src │ ├── test │ │ ├── resources │ │ │ ├── db │ │ │ │ └── migration │ │ │ │ │ └── 1__application_db_stub.sql │ │ │ ├── database │ │ │ │ └── it_database_setup.sql │ │ │ └── application-test.yml │ │ └── java │ │ │ └── org │ │ │ └── zalando │ │ │ └── nakadiproducer │ │ │ ├── TestApplication.java │ │ │ ├── BaseMockedExternalCommunicationIT.java │ │ │ ├── config │ │ │ ├── MockNakadiClientConfig.java │ │ │ └── EmbeddedDataSourceConfig.java │ │ │ ├── EventLockSizeDefaultIT.java │ │ │ ├── util │ │ │ ├── MockPayload.java │ │ │ └── Fixture.java │ │ │ ├── EventLockSizeConfiguredIT.java │ │ │ ├── FlywayDataSourceIT.java │ │ │ ├── NakadiProducerFlywayCallbackIT.java │ │ │ ├── NonNakadiProducerFlywayCallbackIT.java │ │ │ ├── NakadiClientContentEncodingIT.java │ │ │ ├── snapshots │ │ │ ├── SnapshotEventGeneratorAutoconfigurationIT.java │ │ │ └── SnapshotEventGenerationWebEndpointIT.java │ │ │ ├── flowid │ │ │ └── TracerFlowIdComponentTest.java │ │ │ ├── eventlog │ │ │ └── impl │ │ │ │ ├── batcher │ │ │ │ ├── QueryStatementBatcherTest.java │ │ │ │ └── QueryStatementBatcherIT.java │ │ │ │ └── EventLogRepositoryIT.java │ │ │ ├── SubmissionDisabledIT.java │ │ │ ├── LockingIT.java │ │ │ ├── LockTimeoutIT.java │ │ │ └── EndToEndTestIT.java │ └── main │ │ ├── resources │ │ ├── db_nakadiproducer │ │ │ └── migrations │ │ │ │ ├── V2 │ │ │ │ ├── V2133546886.1.2__grant_delete_permissions.sql │ │ │ │ ├── V2133546886.1.1__add_compaction_key.sql │ │ │ │ └── V2133546886.1.0__drop_data_event__specific_columns.sql │ │ │ │ └── V1 │ │ │ │ ├── V1029384757.1.2__schema_permissions.sql │ │ │ │ └── V1029384757.1.1__event_log_table.sql │ │ └── META-INF │ │ │ └── spring.factories │ │ └── java │ │ └── org │ │ └── zalando │ │ └── nakadiproducer │ │ ├── AccessTokenProvider.java │ │ ├── flowid │ │ ├── NoopFlowIdComponent.java │ │ └── TracerFlowIdComponent.java │ │ ├── EnableNakadiProducer.java │ │ ├── NakadiProducerFlywayDataSource.java │ │ ├── StupsTokenComponent.java │ │ ├── EventTransmissionScheduler.java │ │ ├── snapshots │ │ └── impl │ │ │ └── SnapshotEventCreationEndpoint.java │ │ ├── NakadiProducerFlywayCallback.java │ │ ├── eventlog │ │ └── impl │ │ │ └── EventLogRepositoryImpl.java │ │ └── FlywayMigrator.java └── pom.xml ├── .travis.yml ├── .zappr.yaml ├── nakadi-producer ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── zalando │ │ │ └── nakadiproducer │ │ │ ├── flowid │ │ │ └── FlowIdComponent.java │ │ │ ├── transmission │ │ │ ├── NakadiPublishingClient.java │ │ │ ├── impl │ │ │ │ ├── EventTransmitter.java │ │ │ │ ├── NakadiMetadata.java │ │ │ │ ├── FahrscheinNakadiPublishingClient.java │ │ │ │ ├── NakadiEvent.java │ │ │ │ ├── EventBatcher.java │ │ │ │ └── EventTransmissionService.java │ │ │ └── MockNakadiPublishingClient.java │ │ │ ├── snapshots │ │ │ ├── Snapshot.java │ │ │ ├── impl │ │ │ │ ├── SnapshotEventProviderNotImplementedException.java │ │ │ │ └── SnapshotCreationService.java │ │ │ ├── UnknownEventTypeException.java │ │ │ ├── SimpleSnapshotEventGenerator.java │ │ │ └── SnapshotEventGenerator.java │ │ │ └── eventlog │ │ │ ├── impl │ │ │ ├── EventDataOperation.java │ │ │ ├── DataChangeEventEnvelope.java │ │ │ ├── EventLog.java │ │ │ └── EventLogRepository.java │ │ │ ├── CompactionKeyExtractors.java │ │ │ └── CompactionKeyExtractor.java │ └── test │ │ └── java │ │ └── org │ │ └── zalando │ │ └── nakadiproducer │ │ ├── util │ │ ├── MockPayload.java │ │ └── Fixture.java │ │ ├── transmission │ │ ├── MockNakadiPublishingClientTest.java │ │ └── impl │ │ │ └── EventBatcherTest.java │ │ ├── snapshots │ │ └── impl │ │ │ └── SnapshotCreationServiceTest.java │ │ └── eventlog │ │ └── impl │ │ └── EventLogWriterMultipleTypesTest.java └── pom.xml ├── SECURITY.md ├── LICENSE ├── pom.xml ├── CODE_OF_CONDUCT.md ├── mvnw.cmd ├── CONTRIBUTING.md └── mvnw /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | target 4 | .project 5 | .classpath 6 | .settings 7 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # The two maintainers are code owners for everything. 2 | * @fbrns @epaul 3 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/main/resources/db/migration/V1.1__noop.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-type: -------------------------------------------------------------------------------- 1 | Bearer -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-secret: -------------------------------------------------------------------------------- 1 | my fake token -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Paul Ebermann (@ePaul) 2 | Florian Brons (@fbrns) 3 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/main/resources/db/migration/V1__create_nakadi_events_schema.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA IF NOT EXISTS nakadi_events -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalando-nakadi/nakadi-producer-spring-boot-starter/HEAD/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.2.5/apache-maven-3.2.5-bin.zip -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/resources/db/migration/1__application_db_stub.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE dummy ( 2 | id NUMBER NOT NULL 3 | ); -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | install: ./mvnw install -DskipTests=true -Dmaven.javadoc.skip=true -Dgpg.skip=true -B -V 5 | script: ./mvnw verify -Dgpg.skip=true -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/db_nakadiproducer/migrations/V2/V2133546886.1.2__grant_delete_permissions.sql: -------------------------------------------------------------------------------- 1 | GRANT DELETE ON nakadi_events.event_log TO PUBLIC; 2 | -------------------------------------------------------------------------------- /.zappr.yaml: -------------------------------------------------------------------------------- 1 | X-Zalando-Team: lumberjack 2 | X-Zalando-Type: code 3 | 4 | approvals: 5 | groups: 6 | zalando: 7 | minimum: 2 8 | from: 9 | orgs: 10 | - zalando -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/resources/database/it_database_setup.sql: -------------------------------------------------------------------------------- 1 | DROP DATABASE IF EXISTS local_test_nakadi_publisher_db; 2 | 3 | CREATE DATABASE local_test_nakadi_publisher_db; 4 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | org.zalando.nakadiproducer.NakadiProducerAutoConfiguration -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/db_nakadiproducer/migrations/V2/V2133546886.1.1__add_compaction_key.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE nakadi_events.event_log 2 | ADD COLUMN compaction_key TEXT NULL 3 | ; -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | nakadi-producer: 2 | nakadi-base-uri: https://nakadi.example.org:5432 3 | management.endpoints.web.exposure.include: snapshot-event-creation -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/AccessTokenProvider.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | public interface AccessTokenProvider { 4 | String getAccessToken(); 5 | } 6 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/db_nakadiproducer/migrations/V2/V2133546886.1.0__drop_data_event__specific_columns.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE nakadi_events.event_log 2 | DROP COLUMN data_op, 3 | DROP COLUMN data_type; -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/resources/application-test.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | jackson.default-property-inclusion: non_null 3 | jpa: 4 | show-sql: true 5 | 6 | logging.level: 7 | org.springframework.web: 'DEBUG' 8 | 9 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/flowid/FlowIdComponent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.flowid; 2 | 3 | public interface FlowIdComponent { 4 | String getXFlowIdValue(); 5 | 6 | void startTraceIfNoneExists(); 7 | } 8 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/NakadiPublishingClient.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission; 2 | 3 | import java.util.List; 4 | 5 | public interface NakadiPublishingClient { 6 | void publish(String eventType, List nakadiEvents) throws Exception; 7 | } 8 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/Snapshot.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | @AllArgsConstructor 7 | @Getter 8 | public class Snapshot { 9 | private Object id; 10 | private String dataType; 11 | private Object data; 12 | } 13 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/java/org/zalando/nakadiproducer/configuration/AopConfiguration.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.configuration; 2 | 3 | import org.springframework.context.annotation.Configuration; 4 | import org.springframework.context.annotation.EnableAspectJAutoProxy; 5 | 6 | @Configuration 7 | @EnableAspectJAutoProxy 8 | public class AopConfiguration { 9 | } 10 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/db_nakadiproducer/migrations/V1/V1029384757.1.2__schema_permissions.sql: -------------------------------------------------------------------------------- 1 | GRANT USAGE ON SCHEMA nakadi_events TO PUBLIC; 2 | GRANT SELECT ON nakadi_events.event_log TO PUBLIC; 3 | GRANT INSERT ON nakadi_events.event_log TO PUBLIC; 4 | GRANT UPDATE ON nakadi_events.event_log TO PUBLIC; 5 | GRANT USAGE ON SEQUENCE nakadi_events.event_log_id_seq TO PUBLIC; -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventProviderNotImplementedException.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots.impl; 2 | 3 | public class SnapshotEventProviderNotImplementedException extends RuntimeException { 4 | 5 | @Override 6 | public String getMessage() { 7 | return "Snapshot not implemented by the service"; 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/resources/example-business-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "example.business.event", 3 | "owning_application": "nakadi-producer-loadtest", 4 | "category": "undefined", 5 | "partition_strategy": "random", 6 | "schema": { 7 | "type": "json_schema", 8 | "schema": { 9 | "properties": { 10 | "content": { 11 | "type": "string" 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/flowid/NoopFlowIdComponent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.flowid; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | 5 | @Slf4j 6 | public class NoopFlowIdComponent implements FlowIdComponent { 7 | 8 | @Override 9 | public String getXFlowIdValue() { 10 | return null; 11 | } 12 | 13 | @Override 14 | public void startTraceIfNoneExists() { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/TestApplication.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class TestApplication { 8 | 9 | public static void main(final String[] args) { 10 | SpringApplication.run(TestApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | flyway: 2 | schemas: local_nakadi_db 3 | locations: classpath:/db/migration 4 | url: jdbc:postgresql://localhost:5432/local_nakadi_db 5 | username: nakadi 6 | password: nakadi 7 | enabled: true 8 | 9 | nakadi-producer: 10 | nakadi-base-uri: http://localhost:8080 11 | 12 | spring: 13 | datasource: 14 | driverClassName: org.postgresql.Driver 15 | url: jdbc:postgresql://localhost:5432/local_nakadi_db 16 | username: nakadi 17 | password: nakadi -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/impl/EventDataOperation.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | public enum EventDataOperation { 4 | 5 | CREATE("C"), 6 | 7 | UPDATE("U"), 8 | 9 | DELETE("D"), 10 | 11 | SNAPSHOT("S"); 12 | 13 | private final String value; 14 | 15 | EventDataOperation(final String value) { 16 | this.value = value; 17 | } 18 | 19 | @Override 20 | public String toString() { 21 | return value; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/UnknownEventTypeException.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | public class UnknownEventTypeException extends RuntimeException { 4 | 5 | private final String eventType; 6 | 7 | public UnknownEventTypeException(final String eventType) { 8 | this.eventType = eventType; 9 | } 10 | 11 | @Override 12 | public String getMessage() { 13 | return "No event log found for event type (" + eventType + ")."; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/EventTransmitter.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | public class EventTransmitter { 4 | private final EventTransmissionService eventTransmissionService; 5 | 6 | public EventTransmitter(EventTransmissionService eventTransmissionService) { 7 | this.eventTransmissionService = eventTransmissionService; 8 | } 9 | 10 | public void sendEvents() { 11 | eventTransmissionService.sendEvents(eventTransmissionService.lockSomeEvents()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/resources/application.yaml: -------------------------------------------------------------------------------- 1 | flyway: 2 | schemas: local_nakadi_db 3 | locations: classpath:/db/migration 4 | url: jdbc:postgresql://localhost:5432/local_nakadi_db 5 | username: nakadi 6 | password: nakadi 7 | enabled: true 8 | 9 | logging: 10 | level: 11 | org.zalando.nakadiproducer: INFO 12 | 13 | nakadi-producer: 14 | nakadi-base-uri: http://localhost:8080 15 | 16 | spring: 17 | datasource: 18 | driverClassName: org.postgresql.Driver 19 | url: jdbc:postgresql://localhost:5432/local_nakadi_db 20 | username: nakadi 21 | password: nakadi -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/impl/DataChangeEventEnvelope.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | import lombok.Setter; 6 | 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | 9 | @Getter 10 | @Setter 11 | @AllArgsConstructor 12 | class DataChangeEventEnvelope { 13 | @JsonProperty("data_op") 14 | private String dataOp; 15 | 16 | @JsonProperty("data_type") 17 | private String dataType; 18 | 19 | @JsonProperty("data") 20 | private Object data; 21 | } 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | We acknowledge that every line of code that we write may potentially contain security issues. We are trying to deal with it responsibly and provide patches as quickly as possible. 2 | 3 | We host our bug bounty program on HackerOne, it is currently private, therefore if you would like to report a vulnerability and get rewarded for it, please ask to join our program by filling this form: 4 | 5 | https://corporate.zalando.com/en/services-and-contact#security-form 6 | 7 | You can also send your report via this form if you do not want to join our bug bounty program and just want to report a vulnerability or security issue. -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/java/org/zalando/nakadiproducer/configuration/TokenConfiguration.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.configuration; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.context.annotation.Primary; 6 | import org.zalando.nakadiproducer.AccessTokenProvider; 7 | 8 | @Configuration 9 | public class TokenConfiguration { 10 | 11 | @Primary 12 | @Bean 13 | public AccessTokenProvider accessTokenProvider() { 14 | return () -> "MY-FAKE-TOKEN"; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/main/java/org/zalando/nakadiproducer/event/ExampleBusinessEvent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.event; 2 | 3 | public class ExampleBusinessEvent { 4 | 5 | public static String EVENT_NAME = "example.business.event"; 6 | 7 | private String content; 8 | 9 | public ExampleBusinessEvent(String content) { 10 | this.content = content; 11 | } 12 | 13 | public String getContent() { 14 | return content; 15 | } 16 | 17 | @Override 18 | public String toString() { 19 | return "ExampleBusinessEvent{" + "content='" + content + '\'' + '}'; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/EnableNakadiProducer.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import java.lang.annotation.Documented; 4 | import java.lang.annotation.ElementType; 5 | import java.lang.annotation.Retention; 6 | import java.lang.annotation.RetentionPolicy; 7 | import java.lang.annotation.Target; 8 | 9 | /** 10 | * @deprecated This annotation has no effect anymore and will be removed in the next major release 11 | */ 12 | @Target(ElementType.TYPE) 13 | @Retention(RetentionPolicy.RUNTIME) 14 | @Documented 15 | @Deprecated 16 | public @interface EnableNakadiProducer { 17 | } 18 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/MockNakadiConfig.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.tests; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 6 | import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; 7 | 8 | @Configuration 9 | public class MockNakadiConfig { 10 | @Bean 11 | public NakadiPublishingClient mockNakadiPublishingClient() { 12 | return new MockNakadiPublishingClient(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/main/java/org/zalando/nakadiproducer/Application.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.web.client.RestTemplate; 7 | 8 | @EnableAutoConfiguration 9 | public class Application { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(Application.class, args); 13 | } 14 | 15 | @Bean 16 | public RestTemplate restTemplate() { 17 | return new RestTemplate(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/BaseMockedExternalCommunicationIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.springframework.boot.test.context.SpringBootTest; 4 | import org.springframework.test.context.ActiveProfiles; 5 | import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; 6 | 7 | 8 | @ActiveProfiles("test") 9 | @SpringBootTest( 10 | webEnvironment = SpringBootTest.WebEnvironment.MOCK, 11 | properties = { "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false" }, 12 | classes = { TestApplication.class, EmbeddedDataSourceConfig.class } 13 | ) 14 | public abstract class BaseMockedExternalCommunicationIT { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/NakadiMetadata.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import lombok.Data; 4 | 5 | import java.time.Instant; 6 | 7 | import com.fasterxml.jackson.annotation.JsonFormat; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | 10 | @Data 11 | public class NakadiMetadata { 12 | 13 | @JsonProperty("eid") 14 | private String eid; 15 | 16 | @JsonProperty("occurred_at") 17 | @JsonFormat(shape = JsonFormat.Shape.STRING) 18 | private Instant occuredAt; 19 | 20 | @JsonProperty("flow_id") 21 | private String flowId; 22 | 23 | @JsonProperty("partition_compaction_key") 24 | private String partitionCompactionKey; 25 | 26 | } 27 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayDataSource.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import java.lang.annotation.ElementType; 4 | import java.lang.annotation.Retention; 5 | import java.lang.annotation.RetentionPolicy; 6 | import java.lang.annotation.Target; 7 | 8 | import org.springframework.beans.factory.annotation.Qualifier; 9 | 10 | /** 11 | * Qualifier annotation for a DataSource to be used by nakadi-producer's Flyway instance. 12 | */ 13 | @Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.TYPE, 14 | ElementType.ANNOTATION_TYPE }) 15 | @Retention(RetentionPolicy.RUNTIME) 16 | @Qualifier 17 | public @interface NakadiProducerFlywayDataSource { 18 | } 19 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/FahrscheinNakadiPublishingClient.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; 6 | 7 | public class FahrscheinNakadiPublishingClient implements NakadiPublishingClient { 8 | private final org.zalando.fahrschein.NakadiClient delegate; 9 | 10 | public FahrscheinNakadiPublishingClient(org.zalando.fahrschein.NakadiClient delegate) { 11 | this.delegate = delegate; 12 | } 13 | 14 | @Override 15 | public void publish(String eventType, List nakadiEvents) throws Exception { 16 | delegate.publish(eventType, nakadiEvents); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/NakadiEvent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import com.fasterxml.jackson.annotation.*; 4 | import lombok.Data; 5 | 6 | import java.util.HashMap; 7 | 8 | @Data 9 | public class NakadiEvent { 10 | @JsonIgnore 11 | private HashMap data; 12 | 13 | @JsonProperty("metadata") 14 | private NakadiMetadata metadata; 15 | 16 | // "any getter" needed for serialization - we use it to extract the properties of the data object and put them in 17 | // the top level of the serialized JSON, to conform to Nakadi's business event structure 18 | @JsonAnyGetter 19 | public HashMap any() { 20 | return data; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/config/MockNakadiClientConfig.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.context.annotation.Profile; 6 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 7 | import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; 8 | 9 | @Configuration 10 | // we only want this in the tests which actually want a mock, i.e. ones which have a "test" profile. 11 | @Profile("test") 12 | public class MockNakadiClientConfig { 13 | @Bean 14 | public NakadiPublishingClient nakadiClient() { 15 | return new MockNakadiPublishingClient(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/impl/EventLog.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | import lombok.ToString; 9 | 10 | import java.time.Instant; 11 | 12 | @ToString 13 | @Getter 14 | @Setter 15 | @NoArgsConstructor 16 | @AllArgsConstructor 17 | @Builder(toBuilder = true) 18 | public class EventLog { 19 | 20 | private Integer id; 21 | private String eventType; 22 | private String eventBodyData; 23 | private String flowId; 24 | private Instant created; 25 | private Instant lastModified; 26 | private String lockedBy; 27 | private Instant lockedUntil; 28 | private String compactionKey; 29 | } 30 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/resources/db_nakadiproducer/migrations/V1/V1029384757.1.1__event_log_table.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE nakadi_events.event_log ( 2 | id SERIAL PRIMARY KEY NOT NULL, 3 | -- nakadi event type 4 | event_type TEXT NOT NULL, 5 | -- event payload 6 | event_body_data TEXT NOT NULL, 7 | -- e.g. warehouse:event 8 | data_type TEXT, 9 | -- data operation: C, U, D, S 10 | data_op CHAR(1), 11 | flow_id TEXT, 12 | created TIMESTAMPTZ NOT NULL DEFAULT NOW(), 13 | last_modified TIMESTAMPTZ NOT NULL DEFAULT NOW(), 14 | locked_by TEXT, 15 | locked_until TIMESTAMPTZ 16 | ); 17 | 18 | CREATE INDEX event_log_locked_until_index ON nakadi_events.event_log (locked_until); -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/config/EmbeddedDataSourceConfig.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.config; 2 | 3 | 4 | import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.context.annotation.Primary; 8 | 9 | import java.io.IOException; 10 | 11 | import javax.sql.DataSource; 12 | 13 | @Configuration 14 | public class EmbeddedDataSourceConfig { 15 | 16 | @Bean 17 | @Primary 18 | public DataSource dataSource() throws IOException { 19 | return embeddedPostgres().getPostgresDatabase(); 20 | } 21 | 22 | @Bean 23 | public EmbeddedPostgres embeddedPostgres() throws IOException { 24 | return EmbeddedPostgres.start(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/impl/EventLogRepository.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | public interface EventLogRepository { 7 | Collection findByLockedByAndLockedUntilGreaterThan(String lockedBy, Instant lockedUntil); 8 | 9 | void lockSomeMessages(String lockId, Instant now, Instant lockExpires); 10 | 11 | void delete(EventLog eventLog); 12 | 13 | default void delete(Collection eventLogs) { 14 | eventLogs.forEach(this::delete); 15 | } 16 | 17 | void persist(EventLog eventLog); 18 | 19 | default void persist(Collection eventLogs) { 20 | for (EventLog eventLog : eventLogs) { 21 | persist(eventLog); 22 | } 23 | } 24 | 25 | void deleteAll(); 26 | 27 | EventLog findOne(Integer id); 28 | } 29 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/StupsTokenComponent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import java.net.URI; 4 | import java.util.Collection; 5 | 6 | import org.zalando.stups.tokens.AccessTokens; 7 | import org.zalando.stups.tokens.Tokens; 8 | 9 | class StupsTokenComponent implements AccessTokenProvider { 10 | 11 | private static final String TOKEN_ID = "nakadi"; 12 | private AccessTokens accessTokens; 13 | 14 | public StupsTokenComponent(URI accessTokenUri, Collection accessTokenScopes) { 15 | accessTokens = Tokens.createAccessTokensWithUri(accessTokenUri) 16 | .manageToken(TOKEN_ID) 17 | .addScopesTypeSafe(accessTokenScopes) 18 | .done() 19 | .start(); 20 | } 21 | 22 | public void stop() { 23 | accessTokens.stop(); 24 | } 25 | 26 | @Override 27 | public String getAccessToken() { 28 | return accessTokens.get(TOKEN_ID); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/EventTransmissionScheduler.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.springframework.beans.factory.annotation.Value; 4 | import org.springframework.scheduling.annotation.Scheduled; 5 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 6 | 7 | public class EventTransmissionScheduler { 8 | private final EventTransmitter eventTransmitter; 9 | private final boolean scheduledTransmissionEnabled; 10 | 11 | public EventTransmissionScheduler(EventTransmitter eventTransmitter, @Value("${nakadi-producer.scheduled-transmission-enabled:true}") boolean scheduledTransmissionEnabled) { 12 | this.eventTransmitter = eventTransmitter; 13 | this.scheduledTransmissionEnabled = scheduledTransmissionEnabled; 14 | } 15 | 16 | @Scheduled(fixedDelayString = "${nakadi-producer.transmission-polling-delay:1000}") 17 | protected void sendEventsIfSchedulingEnabled() { 18 | if (scheduledTransmissionEnabled) { 19 | eventTransmitter.sendEvents(); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/EventLockSizeDefaultIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static org.hamcrest.MatcherAssert.assertThat; 4 | import static org.hamcrest.Matchers.hasSize; 5 | 6 | import org.junit.jupiter.api.Test; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 9 | import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; 10 | import org.zalando.nakadiproducer.util.Fixture; 11 | 12 | public class EventLockSizeDefaultIT extends BaseMockedExternalCommunicationIT { 13 | 14 | @Autowired 15 | private EventLogWriter eventLogWriter; 16 | 17 | @Autowired 18 | private EventTransmissionService eventTransmissionService; 19 | 20 | @Test 21 | public void defaultEventLockSizeIsUsed() { 22 | 23 | for (int i = 1; i <= 8; i++) { 24 | eventLogWriter.fireBusinessEvent("myEventType", Fixture.mockPayload(i, "code123")); 25 | } 26 | 27 | assertThat(eventTransmissionService.lockSomeEvents(), hasSize(8)); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/java/org/zalando/nakadiproducer/interceptor/ProfilerInterceptor.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.interceptor; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.aspectj.lang.ProceedingJoinPoint; 5 | import org.aspectj.lang.annotation.Around; 6 | import org.aspectj.lang.annotation.Aspect; 7 | import org.springframework.util.StopWatch; 8 | 9 | @Aspect 10 | @Slf4j 11 | public class ProfilerInterceptor { 12 | 13 | @Around("execution(* org.zalando.nakadiproducer.transmission.impl.EventTransmissionService.lockSomeEvents(..)) || " + 14 | "execution(* org.zalando.nakadiproducer.transmission.impl.EventTransmissionService.sendEvents(..))") 15 | public Object profile(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { 16 | StopWatch clock = new StopWatch("Profiling for " + proceedingJoinPoint.toShortString()); 17 | try { 18 | clock.start(proceedingJoinPoint.toShortString()); 19 | return proceedingJoinPoint.proceed(); 20 | } finally { 21 | clock.stop(); 22 | log.info(clock.prettyPrint()); 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Zalando SE 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/util/MockPayload.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.util; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | import lombok.ToString; 9 | 10 | import java.util.List; 11 | 12 | @ToString 13 | @Getter 14 | @Setter 15 | @NoArgsConstructor 16 | @AllArgsConstructor 17 | @Builder(toBuilder = true) 18 | public class MockPayload { 19 | private Integer id; 20 | 21 | private String code; 22 | 23 | private boolean isActive = true; 24 | 25 | private SubClass more; 26 | 27 | private List items; 28 | 29 | @Builder(toBuilder = true) 30 | @Getter 31 | @Setter 32 | @ToString 33 | @AllArgsConstructor 34 | @NoArgsConstructor 35 | public static class SubClass { 36 | private String info; 37 | } 38 | 39 | @Builder(toBuilder = true) 40 | @Getter 41 | @Setter 42 | @ToString 43 | @AllArgsConstructor 44 | @NoArgsConstructor 45 | public static class SubListItem { 46 | private String detail; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/README.md: -------------------------------------------------------------------------------- 1 | # nakadi-producer-loadtest 2 | 3 | The project contains functionality to create load for nakadi-producer (10k, 50k, 100k and 300k events) and measures 4 | the execution time. The events are fired against an internally started nakadi instance. Therefore it uses docker-compose 5 | to start kafka, nakadi, postgres and zookeeper. 6 | 7 | #### Prerequisites 8 | 9 | [docker-compose](https://docs.docker.com/compose/) must be installed. The docker images will be pulled automatically by 10 | executing the integration test or can be manually pulled by executing 11 | ``` 12 | docker-compose -f nakadi-producer-loadtest/src/test/resources/docker-compose.yaml pull 13 | ``` 14 | 15 | #### Usage 16 | ``` 17 | mvn test -Dtest=LoadTestIT -Dgpg.skip=true 18 | ``` 19 | 20 | #### Configuration 21 | 22 | The time-tracking functionality can be configured by changing pointcuts in `ProfilerInterceptor.java`. 23 | ``` 24 | @Around("execution(* org.zalando.nakadiproducer.transmission.impl.EventTransmissionService.lockSomeEvents(..)) || " + 25 | "execution(* org.zalando.nakadiproducer.transmission.impl.EventTransmissionService.sendEvents(..))") 26 | ``` -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/util/MockPayload.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.util; 2 | 3 | import java.util.List; 4 | 5 | import lombok.AllArgsConstructor; 6 | import lombok.Builder; 7 | import lombok.Getter; 8 | import lombok.NoArgsConstructor; 9 | import lombok.Setter; 10 | import lombok.ToString; 11 | 12 | @ToString 13 | @Getter 14 | @Setter 15 | @NoArgsConstructor 16 | @AllArgsConstructor 17 | @Builder(toBuilder = true) 18 | public class MockPayload { 19 | private Integer id; 20 | 21 | private String code; 22 | 23 | @Builder.Default 24 | private boolean isActive = true; 25 | 26 | private SubClass more; 27 | 28 | private List items; 29 | 30 | @Builder(toBuilder = true) 31 | @Getter 32 | @Setter 33 | @ToString 34 | @AllArgsConstructor 35 | @NoArgsConstructor 36 | public static class SubClass { 37 | private String info; 38 | } 39 | 40 | @Builder(toBuilder = true) 41 | @Getter 42 | @Setter 43 | @ToString 44 | @AllArgsConstructor 45 | @NoArgsConstructor 46 | public static class SubListItem { 47 | private String detail; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/resources/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | 4 | nakadi: 5 | image: adyach/nakadi-docker:3.0.8 6 | ports: 7 | - "8080:8080" 8 | depends_on: 9 | - postgres_nakadi 10 | - zookeeper 11 | - kafka 12 | environment: 13 | - SPRING_PROFILES_ACTIVE=local 14 | - NAKADI_OAUTH2_MODE=OFF 15 | - NAKADI_ZOOKEEPER_BROKERS=zookeeper:2181 16 | - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres_nakadi:5432/local_nakadi_db 17 | 18 | postgres_nakadi: 19 | image: adyach/nakadi-postgres:latest 20 | ports: 21 | - "5432:5432" 22 | environment: 23 | POSTGRES_USER: nakadi 24 | POSTGRES_PASSWORD: nakadi 25 | POSTGRES_DB: local_nakadi_db 26 | 27 | zookeeper: 28 | image: wurstmeister/zookeeper:3.4.6 29 | ports: 30 | - "2181:2181" 31 | 32 | kafka: 33 | image: wurstmeister/kafka:0.10.1.0 34 | ports: 35 | - "9092:9092" 36 | depends_on: 37 | - zookeeper 38 | environment: 39 | KAFKA_ADVERTISED_HOST_NAME: kafka 40 | KAFKA_ADVERTISED_PORT: 9092 41 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 42 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' 43 | KAFKA_DELETE_TOPIC_ENABLE: 'true' 44 | KAFKA_BROKER_ID: 0 45 | volumes: 46 | - /var/run/docker.sock:/var/run/docker.sock -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/EventLockSizeConfiguredIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static org.hamcrest.MatcherAssert.assertThat; 4 | import static org.hamcrest.Matchers.hasSize; 5 | 6 | import org.junit.jupiter.api.Test; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.test.context.SpringBootTest; 9 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 10 | import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; 11 | import org.zalando.nakadiproducer.util.Fixture; 12 | 13 | @SpringBootTest( 14 | properties = {"nakadi-producer.lock-size=3"} 15 | ) 16 | public class EventLockSizeConfiguredIT extends BaseMockedExternalCommunicationIT { 17 | 18 | @Autowired 19 | private EventLogWriter eventLogWriter; 20 | 21 | @Autowired 22 | private EventTransmissionService eventTransmissionService; 23 | 24 | @Test 25 | public void eventLockSizeIsRespected() { 26 | 27 | for (int i = 1; i <= 8; i++) { 28 | eventLogWriter.fireBusinessEvent( "myEventType", Fixture.mockPayload(i, "code123")); 29 | } 30 | 31 | assertThat(eventTransmissionService.lockSomeEvents(), hasSize(3)); 32 | assertThat(eventTransmissionService.lockSomeEvents(), hasSize(3)); 33 | assertThat(eventTransmissionService.lockSomeEvents(), hasSize(2)); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/FlywayDataSourceIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static org.mockito.Mockito.atLeastOnce; 4 | import static org.mockito.Mockito.verify; 5 | 6 | import java.sql.SQLException; 7 | 8 | import javax.sql.DataSource; 9 | 10 | import org.junit.jupiter.api.Test; 11 | import org.mockito.Mockito; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.context.annotation.Bean; 14 | import org.springframework.context.annotation.Configuration; 15 | import org.springframework.test.context.ContextConfiguration; 16 | 17 | import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; 18 | 19 | @ContextConfiguration(classes=FlywayDataSourceIT.Config.class) 20 | public class FlywayDataSourceIT extends BaseMockedExternalCommunicationIT { 21 | @Autowired 22 | @NakadiProducerFlywayDataSource 23 | private DataSource dataSource; 24 | 25 | @Test 26 | public void usesNakadiProducerDataSourceIfAnnotatedWithQualifier() throws SQLException { 27 | verify(dataSource, atLeastOnce()).getConnection(); 28 | } 29 | 30 | @Configuration 31 | public static class Config { 32 | @Autowired 33 | EmbeddedPostgres embeddedPostgres; 34 | 35 | @Bean 36 | @NakadiProducerFlywayDataSource 37 | public DataSource flywayDataSource() { 38 | return Mockito.spy(embeddedPostgres.getPostgresDatabase()); 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/CompactionKeyExtractors.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | import java.util.Optional; 8 | import java.util.function.Function; 9 | 10 | /** 11 | * This class contains implementations of {@link CompactionKeyExtractor} used by the factory methods in that interface. 12 | */ 13 | final class CompactionKeyExtractors { 14 | 15 | @AllArgsConstructor(access = AccessLevel.PACKAGE) 16 | static class SimpleCompactionKeyExtractor implements CompactionKeyExtractor { 17 | @Getter 18 | private final String eventType; 19 | private final Function> extractorFunction; 20 | 21 | @Override 22 | public Optional tryGetKeyFor(Object o) { 23 | return extractorFunction.apply(o); 24 | } 25 | } 26 | 27 | @AllArgsConstructor(access = AccessLevel.PACKAGE) 28 | static class TypedCompactionKeyExtractor implements CompactionKeyExtractor { 29 | @Getter 30 | private final String eventType; 31 | private final Class type; 32 | private final Function extractorFunction; 33 | 34 | @Override 35 | public Optional tryGetKeyFor(Object o) { 36 | if(type.isInstance(o)) { 37 | return Optional.of(extractorFunction.apply(type.cast(o))); 38 | } else { 39 | return Optional.empty(); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static org.mockito.Matchers.any; 4 | import static org.mockito.Mockito.inOrder; 5 | import static org.mockito.Mockito.times; 6 | import static org.mockito.Mockito.verify; 7 | 8 | import java.sql.Connection; 9 | 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.boot.test.mock.mockito.MockBean; 12 | import org.springframework.test.annotation.DirtiesContext; 13 | 14 | public class NakadiProducerFlywayCallbackIT extends BaseMockedExternalCommunicationIT { 15 | 16 | @MockBean 17 | private NakadiProducerFlywayCallback flywayCallback; 18 | 19 | @MockBean 20 | private ConfigurationAwareNakadiProducerFlywayCallback configurationAwareNakadiProducerFlywayCallback; 21 | 22 | @Test 23 | @DirtiesContext // Needed to make sure that flyway gets executed for each of the tests and Callbacks are called again 24 | public void flywayCallbackIsCalledIfAnnotatedWithQualifierAnnotation() { 25 | verify(flywayCallback, times(1)).beforeMigrate(any(Connection.class)); 26 | } 27 | 28 | @Test 29 | @DirtiesContext // Needed to make sure that flyway gets executed for each of the tests and Callbacks are called again 30 | public void flywayConfigurationIsSetIfCallbackIsConfigurationAware() { 31 | verify(configurationAwareNakadiProducerFlywayCallback, times(1)).beforeMigrate(any(Connection.class)); 32 | } 33 | 34 | public interface ConfigurationAwareNakadiProducerFlywayCallback extends NakadiProducerFlywayCallback { 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static org.mockito.Matchers.any; 4 | import static org.mockito.Mockito.never; 5 | import static org.mockito.Mockito.verify; 6 | 7 | import java.sql.Connection; 8 | 9 | import org.junit.jupiter.api.Test; 10 | import org.springframework.boot.test.context.SpringBootTest; 11 | import org.springframework.boot.test.mock.mockito.MockBean; 12 | import org.springframework.test.context.ActiveProfiles; 13 | import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; 14 | 15 | @ActiveProfiles("test") 16 | @SpringBootTest( 17 | webEnvironment = SpringBootTest.WebEnvironment.MOCK, 18 | properties = {"zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", "spring.flyway.enabled:false"}, 19 | classes = {TestApplication.class, EmbeddedDataSourceConfig.class} 20 | ) 21 | public class NonNakadiProducerFlywayCallbackIT { 22 | 23 | @MockBean 24 | private NakadiProducerFlywayCallback flywayCallback; 25 | 26 | @Test 27 | public void flywayCallbacksFromOurHostApplicationAreNotUsedByUs() { 28 | verify(flywayCallback, never()).beforeValidate(any(Connection.class)); 29 | } 30 | 31 | @Test 32 | public void ourOwnFlywayConfigurationStillWorksFineWhenSpringsFlywayAutoconfigIsDisabled() { 33 | // Yes, this is redundant to the other test in here. 34 | // We consider it important to document the requirement, so it is here nonetheless. 35 | // The test setup done by the class annotations does just enough to test it 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiClientContentEncodingIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import lombok.SneakyThrows; 4 | import org.apache.commons.lang3.reflect.FieldUtils; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.test.context.SpringBootTest; 8 | import org.springframework.boot.test.mock.mockito.MockBean; 9 | import org.zalando.fahrschein.http.api.ContentEncoding; 10 | import org.zalando.fahrschein.http.api.RequestFactory; 11 | import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; 12 | 13 | import static org.hamcrest.MatcherAssert.assertThat; 14 | import static org.hamcrest.Matchers.is; 15 | 16 | // this class has no @ActiveProfiles("test"), so it doesn't use the MockNakadiClient. 17 | @SpringBootTest( 18 | webEnvironment = SpringBootTest.WebEnvironment.MOCK, 19 | properties = { 20 | "nakadi-producer.scheduled-transmission-enabled:false", 21 | // as we are not defining a mock nakadi client, we need to provide these properties: 22 | "nakadi-producer.encoding:ZSTD", 23 | "nakadi-producer.nakadi-base-uri:http://nakadi.example.com/", 24 | }, 25 | classes = { TestApplication.class, EmbeddedDataSourceConfig.class } 26 | ) 27 | public class NakadiClientContentEncodingIT { 28 | 29 | // Avoid errors in the logs from the AccessTokenRefresher. As we are not actually submitting 30 | // to Nakadi, this will never be used. 31 | @MockBean 32 | private AccessTokenProvider tokenProvider; 33 | 34 | @Autowired 35 | private RequestFactory requestFactory; 36 | 37 | @Test 38 | @SneakyThrows 39 | public void pickUpContentEncodingFromConfig() { 40 | final ContentEncoding contentEncoding = 41 | (ContentEncoding) FieldUtils.readField(requestFactory, "contentEncoding", true); 42 | assertThat(contentEncoding, is(ContentEncoding.ZSTD)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGeneratorAutoconfigurationIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | import static org.assertj.core.api.Fail.fail; 4 | 5 | import java.util.Collections; 6 | 7 | import org.junit.jupiter.api.Test; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.zalando.nakadiproducer.BaseMockedExternalCommunicationIT; 13 | import org.zalando.nakadiproducer.snapshots.impl.SnapshotCreationService; 14 | 15 | @ContextConfiguration(classes = SnapshotEventGeneratorAutoconfigurationIT.Config.class) 16 | public class SnapshotEventGeneratorAutoconfigurationIT extends BaseMockedExternalCommunicationIT { 17 | 18 | @Autowired 19 | private SnapshotCreationService snapshotCreationService; 20 | 21 | @Test 22 | public void picksUpDefinedSnapshotEventProviders() { 23 | // expect no exceptions 24 | snapshotCreationService.createSnapshotEvents("A", ""); 25 | 26 | // expect no exceptions 27 | snapshotCreationService.createSnapshotEvents("B", ""); 28 | 29 | try { 30 | snapshotCreationService.createSnapshotEvents("not defined", ""); 31 | } catch (final UnknownEventTypeException e) { 32 | return; 33 | } 34 | 35 | fail("unknown event type did not result in an exception"); 36 | } 37 | 38 | @Configuration 39 | public static class Config { 40 | 41 | @Bean 42 | public SnapshotEventGenerator snapshotEventProviderA() { 43 | return new SimpleSnapshotEventGenerator("A", (x) -> Collections.emptyList()); 44 | } 45 | 46 | @Bean 47 | public SnapshotEventGenerator snapshotEventProviderB() { 48 | return new SimpleSnapshotEventGenerator("B", (x) -> Collections.emptyList()); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.tests; 2 | 3 | import com.opentable.db.postgres.embedded.EmbeddedPostgres; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Primary; 8 | import org.zalando.nakadiproducer.EnableNakadiProducer; 9 | import org.zalando.nakadiproducer.snapshots.SimpleSnapshotEventGenerator; 10 | import org.zalando.nakadiproducer.snapshots.Snapshot; 11 | import org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator; 12 | 13 | import javax.sql.DataSource; 14 | 15 | import java.io.IOException; 16 | import java.util.ArrayList; 17 | import java.util.Collections; 18 | 19 | @EnableAutoConfiguration 20 | @EnableNakadiProducer 21 | public class Application { 22 | 23 | public static void main(String[] args) throws Exception { 24 | SpringApplication.run(Application.class, args); 25 | } 26 | 27 | @Bean 28 | @Primary 29 | public DataSource dataSource(EmbeddedPostgres postgres) throws IOException { 30 | return postgres.getPostgresDatabase(); 31 | } 32 | 33 | @Bean 34 | public EmbeddedPostgres embeddedPostgres() throws IOException { 35 | return EmbeddedPostgres.start(); 36 | } 37 | 38 | @Bean 39 | public SnapshotEventGenerator snapshotEventGenerator() { 40 | return new SimpleSnapshotEventGenerator("eventtype", (withIdGreaterThan, filter) -> { 41 | if (withIdGreaterThan == null) { 42 | return Collections.singletonList(new Snapshot("1", "foo", filter)); 43 | } else if (withIdGreaterThan.equals("1")) { 44 | return Collections.singletonList(new Snapshot("2", "foo", filter)); 45 | } else { 46 | return new ArrayList<>(); 47 | } 48 | }); 49 | 50 | // Todo: Test that some events arrive at a local nakadi mock 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots.impl; 2 | 3 | import java.util.Set; 4 | 5 | import org.springframework.boot.actuate.endpoint.annotation.Endpoint; 6 | import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; 7 | import org.springframework.boot.actuate.endpoint.annotation.Selector; 8 | import org.springframework.boot.actuate.endpoint.annotation.WriteOperation; 9 | 10 | import lombok.AllArgsConstructor; 11 | import lombok.Getter; 12 | import org.springframework.lang.Nullable; 13 | import org.zalando.nakadiproducer.flowid.FlowIdComponent; 14 | 15 | @Endpoint(id = "snapshot-event-creation") 16 | public class SnapshotEventCreationEndpoint { 17 | private final SnapshotCreationService snapshotCreationService; 18 | private final FlowIdComponent flowIdComponent; 19 | 20 | public SnapshotEventCreationEndpoint(SnapshotCreationService snapshotCreationService, FlowIdComponent flowIdComponent) { 21 | this.snapshotCreationService = snapshotCreationService; 22 | this.flowIdComponent = flowIdComponent; 23 | } 24 | 25 | @ReadOperation 26 | public SnapshotReport getSupportedEventTypes() { 27 | return new SnapshotReport(snapshotCreationService.getSupportedEventTypes()); 28 | } 29 | 30 | @WriteOperation 31 | public void createFilteredSnapshotEvents( 32 | // this is the event type. Could have a better name, but since Spring Boot relies on the -parameters 33 | // compiler flag being set to resolve path parameter names, it would then get trickier to reliably run this 34 | // Test in the IDE. So let's stick with arg0 for now. 35 | @Selector String arg0, 36 | @Nullable String filter) { 37 | flowIdComponent.startTraceIfNoneExists(); 38 | snapshotCreationService.createSnapshotEvents(arg0, filter); 39 | } 40 | 41 | 42 | @AllArgsConstructor 43 | @Getter 44 | public static class SnapshotReport { 45 | private final Set supportedEventTypes; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/flowid/TracerFlowIdComponent.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.flowid; 2 | 3 | import org.zalando.tracer.Tracer; 4 | 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | @Slf4j 8 | public class TracerFlowIdComponent implements FlowIdComponent { 9 | 10 | private static final String X_FLOW_ID = "X-Flow-ID"; 11 | 12 | private final Tracer tracer; 13 | 14 | public TracerFlowIdComponent(Tracer tracer) { 15 | this.tracer = tracer; 16 | } 17 | 18 | public String getXFlowIdKey() { 19 | return X_FLOW_ID; 20 | } 21 | 22 | @Override 23 | public String getXFlowIdValue() { 24 | if (tracer != null) { 25 | try { 26 | return tracer.get(X_FLOW_ID).getValue(); 27 | } catch (IllegalArgumentException e) { 28 | log.warn("No trace was configured for the name {}. Returning null. " + 29 | "To configure Tracer provide an application property: " + 30 | "tracer.traces.X-Flow-ID=flow-id", X_FLOW_ID); 31 | } catch (IllegalStateException e) { 32 | log.warn("Unexpected Error while receiving the Trace Id {}. Returning null. " + 33 | "Please check your tracer configuration: {}", X_FLOW_ID, e.getMessage()); 34 | } 35 | } else { 36 | log.warn("No bean of class Tracer was found. Returning null."); 37 | } 38 | return null; 39 | } 40 | 41 | @Override 42 | public void startTraceIfNoneExists() { 43 | if (tracer != null) { 44 | try { 45 | tracer.get(X_FLOW_ID).getValue(); 46 | } catch (IllegalArgumentException e) { 47 | log.warn("No trace was configured for the name {}. Returning null. " + 48 | "To configure Tracer provide an application property: " + 49 | "tracer.traces.X-Flow-ID=flow-id", X_FLOW_ID); 50 | } catch (IllegalStateException e) { 51 | tracer.start(); 52 | } 53 | } else { 54 | log.warn("No bean of class Tracer was found."); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/flowid/TracerFlowIdComponentTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.flowid; 2 | 3 | import static org.hamcrest.MatcherAssert.assertThat; 4 | import static org.mockito.Mockito.never; 5 | import static org.mockito.Mockito.verify; 6 | import static org.mockito.Mockito.when; 7 | 8 | import org.hamcrest.Matchers; 9 | import org.junit.jupiter.api.Test; 10 | import org.junit.jupiter.api.extension.ExtendWith; 11 | import org.mockito.Answers; 12 | import org.mockito.Mock; 13 | import org.mockito.junit.jupiter.MockitoExtension; 14 | import org.zalando.tracer.Tracer; 15 | 16 | @ExtendWith(MockitoExtension.class) 17 | public class TracerFlowIdComponentTest { 18 | 19 | @Mock(answer = Answers.RETURNS_DEEP_STUBS) 20 | private Tracer tracer; 21 | 22 | @Test 23 | public void makeSureItWorks() { 24 | TracerFlowIdComponent flowIdComponent = new TracerFlowIdComponent(tracer); 25 | when(tracer.get("X-Flow-ID").getValue()).thenReturn("A_FUNKY_VALUE"); 26 | 27 | assertThat(flowIdComponent.getXFlowIdKey(), Matchers.equalTo("X-Flow-ID")); 28 | assertThat(flowIdComponent.getXFlowIdValue(), Matchers.equalTo("A_FUNKY_VALUE")); 29 | } 30 | 31 | @Test 32 | public void makeSureTraceWillBeStartedIfNoneHasBeenStartedBefore() { 33 | TracerFlowIdComponent flowIdComponent = new TracerFlowIdComponent(tracer); 34 | when(tracer.get("X-Flow-ID").getValue()).thenThrow(new IllegalStateException()); 35 | 36 | flowIdComponent.startTraceIfNoneExists(); 37 | 38 | verify(tracer).start(); 39 | } 40 | 41 | @Test 42 | public void wontFailIfTraceHasNotBeenConfiguredInStartTrace() { 43 | TracerFlowIdComponent flowIdComponent = new TracerFlowIdComponent(tracer); 44 | when(tracer.get("X-Flow-ID")).thenThrow(new IllegalArgumentException()); 45 | 46 | flowIdComponent.startTraceIfNoneExists(); 47 | 48 | // then no exception is thrown 49 | } 50 | 51 | @Test 52 | public void makeSureTraceWillNotStartedIfOneExists() { 53 | TracerFlowIdComponent flowIdComponent = new TracerFlowIdComponent(tracer); 54 | when(tracer.get("X-Flow-ID").getValue()).thenReturn("A_FUNKY_VALUE"); 55 | 56 | flowIdComponent.startTraceIfNoneExists(); 57 | 58 | verify(tracer, never()).start(); 59 | } 60 | 61 | } -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/util/Fixture.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.zalando.nakadiproducer.snapshots.Snapshot; 7 | 8 | public class Fixture { 9 | 10 | public static final String PUBLISHER_EVENT_TYPE = "wholesale.some-publisher-change-event"; 11 | public static final String PUBLISHER_DATA_TYPE = "nakadi:some-publisher"; 12 | 13 | public static MockPayload mockPayload(Integer id, String code, Boolean isActive, 14 | MockPayload.SubClass more, List items) { 15 | return MockPayload.builder() 16 | .id(id) 17 | .code(code) 18 | .isActive(isActive) 19 | .more(more) 20 | .items(items) 21 | .build(); 22 | } 23 | 24 | public static MockPayload mockPayload(Integer id, String code) { 25 | return mockPayload(id, code, true, mockSubClass(), mockSubList(3)); 26 | } 27 | 28 | public static List mockSnapshotList(Integer size) { 29 | final List list = new ArrayList<>(); 30 | for (int i = 0; i < size; i++) { 31 | list.add(new Snapshot(i, PUBLISHER_DATA_TYPE, mockPayload(i + 1, "code" + i, true, 32 | mockSubClass("some info " + i), mockSubList(3, "some detail for code" + i)))); 33 | } 34 | return list; 35 | } 36 | 37 | public static MockPayload.SubClass mockSubClass(String info) { 38 | return MockPayload.SubClass.builder().info(info).build(); 39 | } 40 | 41 | private static MockPayload.SubClass mockSubClass() { 42 | return mockSubClass("Info something"); 43 | } 44 | 45 | private static MockPayload.SubListItem mockSubListItem(String detail) { 46 | return MockPayload.SubListItem.builder().detail(detail).build(); 47 | } 48 | 49 | public static List mockSubList(Integer size, String detail) { 50 | final List items = new ArrayList<>(); 51 | for (int i = 0; i < size; i++) { 52 | items.add(mockSubListItem(detail + i)); 53 | } 54 | return items; 55 | } 56 | 57 | private static List mockSubList(Integer size) { 58 | return mockSubList(size, "Detail something "); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/eventlog/impl/batcher/QueryStatementBatcherTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl.batcher; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.hamcrest.MatcherAssert.assertThat; 6 | import static org.hamcrest.Matchers.hasSize; 7 | import static org.hamcrest.Matchers.is; 8 | 9 | public class QueryStatementBatcherTest 10 | { 11 | @Test 12 | public void testComposeTemplateInsert() { 13 | 14 | String prefix = "INSERT INTO x (a, b) VALUES "; 15 | String repeated = "(:a#, :b#)"; 16 | String placeholder = "#"; 17 | String separator = ", "; 18 | String suffix = " RETURNING id"; 19 | 20 | assertThat(QueryStatementBatcher.composeTemplate(1, prefix, repeated, placeholder, separator, suffix), 21 | is("INSERT INTO x (a, b) VALUES (:a0, :b0) RETURNING id")); 22 | assertThat(QueryStatementBatcher.composeTemplate(2, prefix, repeated, placeholder, separator, suffix), 23 | is("INSERT INTO x (a, b) VALUES (:a0, :b0), (:a1, :b1) RETURNING id")); 24 | } 25 | 26 | @Test 27 | public void testComposeTemplateSelectWhere() { 28 | String prefix = "SELECT a, b FROM x WHERE id IN ("; 29 | String repeated = ":id#"; 30 | String separator = ", "; 31 | String placeholder = "#"; 32 | String suffix = ")"; 33 | 34 | assertThat(QueryStatementBatcher.composeTemplate(1, prefix, repeated, placeholder, separator, suffix), 35 | is("SELECT a, b FROM x WHERE id IN (:id0)")); 36 | assertThat(QueryStatementBatcher.composeTemplate(2, prefix, repeated, placeholder, separator, suffix), 37 | is("SELECT a, b FROM x WHERE id IN (:id0, :id1)")); 38 | } 39 | 40 | @Test 41 | public void testCreateSubTemplates() { 42 | QueryStatementBatcher batcher = new QueryStatementBatcher<>( 43 | "SELECT a, b FROM x WHERE id IN (", ":id#", ")", 44 | (row, n) -> null, 45 | 21, 6, 1); 46 | assertThat(batcher.subTemplates, hasSize(3)); 47 | assertThat(batcher.subTemplates.get(0).expandedTemplate, 48 | is("SELECT a, b FROM x WHERE id IN (:id0, :id1, :id2, :id3, :id4, :id5, :id6, :id7," + 49 | " :id8, :id9, :id10, :id11, :id12, :id13, :id14, :id15, :id16, :id17, :id18, :id19, :id20)") ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/util/Fixture.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.zalando.nakadiproducer.snapshots.Snapshot; 7 | 8 | public class Fixture { 9 | 10 | public static final String PUBLISHER_EVENT_TYPE = "wholesale.some-publisher-change-event"; 11 | public static final String PUBLISHER_DATA_TYPE = "nakadi:some-publisher"; 12 | 13 | public static MockPayload mockPayload(Integer id, String code, Boolean isActive, 14 | MockPayload.SubClass more, List items) { 15 | return MockPayload.builder() 16 | .id(id) 17 | .code(code) 18 | .isActive(isActive) 19 | .more(more) 20 | .items(items) 21 | .build(); 22 | } 23 | 24 | public static MockPayload mockPayload(Integer id, String code) { 25 | return mockPayload(id, code, true, mockSubClass(), mockSubList(3)); 26 | } 27 | 28 | public static List mockSnapshotList(Integer size) { 29 | final List list = new ArrayList<>(); 30 | for (int i = 0; i < size; i++) { 31 | list.add(new Snapshot(i, PUBLISHER_DATA_TYPE, mockPayload(i + 1, "code" + i, true, 32 | mockSubClass("some info " + i), mockSubList(3, "some detail for code" + i)))); 33 | } 34 | return list; 35 | } 36 | 37 | public static MockPayload.SubClass mockSubClass(String info) { 38 | return MockPayload.SubClass.builder().info(info).build(); 39 | } 40 | 41 | private static MockPayload.SubClass mockSubClass() { 42 | return mockSubClass("Info something"); 43 | } 44 | 45 | private static MockPayload.SubListItem mockSubListItem(String detail) { 46 | return MockPayload.SubListItem.builder().detail(detail).build(); 47 | } 48 | 49 | public static List mockSubList(Integer size, String detail) { 50 | final List items = new ArrayList<>(); 51 | for (int i = 0; i < size; i++) { 52 | items.add(mockSubListItem(detail + i)); 53 | } 54 | return items; 55 | } 56 | 57 | private static List mockSubList(Integer size) { 58 | return mockSubList(size, "Detail something "); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | nakadi-producer-loadtest 8 | ${project.parent.version} 9 | 10 | 11 | org.zalando 12 | nakadi-producer-reactor 13 | 21.0.0 14 | 15 | 16 | 17 | 18 | org.zalando 19 | nakadi-producer-spring-boot-starter 20 | ${project.version} 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-web 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-jdbc 29 | 30 | 31 | org.postgresql 32 | postgresql 33 | 42.3.8 34 | 35 | 36 | org.zalando.stups 37 | tokens 38 | 0.14.0 39 | 40 | 41 | org.springframework.boot 42 | spring-boot-starter-test 43 | test 44 | 45 | 46 | org.springframework 47 | spring-aspects 48 | test 49 | 50 | 51 | org.testcontainers 52 | testcontainers 53 | 1.16.0 54 | test 55 | 56 | 57 | org.projectlombok 58 | lombok 59 | 1.18.22 60 | provided 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/MockNakadiPublishingClient.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.core.JsonProcessingException; 5 | import com.fasterxml.jackson.databind.DeserializationFeature; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.fasterxml.jackson.databind.PropertyNamingStrategies; 8 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 9 | import org.springframework.util.LinkedMultiValueMap; 10 | import org.springframework.util.MultiValueMap; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | public class MockNakadiPublishingClient implements NakadiPublishingClient { 16 | private final ObjectMapper objectMapper; 17 | private final MultiValueMap sentEvents = new LinkedMultiValueMap<>(); 18 | 19 | public MockNakadiPublishingClient() { 20 | this(createDefaultObjectMapper()); 21 | } 22 | 23 | public MockNakadiPublishingClient(ObjectMapper objectMapper) { 24 | this.objectMapper = objectMapper; 25 | } 26 | 27 | @Override 28 | public synchronized void publish(String eventType, List nakadiEvents) throws Exception { 29 | nakadiEvents.stream().map(this::transformToJson).forEach(e -> sentEvents.add(eventType, e)); 30 | } 31 | 32 | public synchronized List getSentEvents(String eventType) { 33 | ArrayList events = new ArrayList<>(); 34 | List sentEvents = this.sentEvents.get(eventType); 35 | if (sentEvents != null) { 36 | events.addAll(sentEvents); 37 | } 38 | return events; 39 | } 40 | 41 | public synchronized void clearSentEvents() { 42 | sentEvents.clear(); 43 | } 44 | 45 | private String transformToJson(Object value) { 46 | try { 47 | return objectMapper.writeValueAsString(value); 48 | } catch (JsonProcessingException e) { 49 | throw new RuntimeException(e); 50 | } 51 | } 52 | 53 | private static ObjectMapper createDefaultObjectMapper() { 54 | final ObjectMapper objectMapper = new ObjectMapper(); 55 | objectMapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); 56 | objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); 57 | objectMapper.registerModules(new JavaTimeModule()); 58 | objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); 59 | return objectMapper; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.tests; 2 | 3 | import org.junit.*; 4 | import org.junit.contrib.java.lang.system.EnvironmentVariables; 5 | import org.junit.runner.RunWith; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.actuate.autoconfigure.web.server.LocalManagementPort; 8 | import org.springframework.boot.test.context.SpringBootTest; 9 | import org.springframework.test.context.junit4.SpringRunner; 10 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 11 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 12 | 13 | import java.io.File; 14 | import java.util.List; 15 | 16 | import static io.restassured.RestAssured.given; 17 | import static org.hamcrest.Matchers.hasSize; 18 | import static org.junit.Assert.assertThat; 19 | 20 | @RunWith(SpringRunner.class) 21 | @SpringBootTest( 22 | // This line looks like that by intention: We want to test that the MockNakadiPublishingClient will be picked up 23 | // by our starter *even if* it has been defined *after* the application itself. This has been a problem until 24 | // this commit. 25 | classes = { Application.class, MockNakadiConfig.class }, 26 | properties = { "nakadi-producer.transmission-polling-delay=30"}, 27 | webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT 28 | ) 29 | public class ApplicationIT { 30 | @LocalManagementPort 31 | private int localManagementPort; 32 | 33 | @ClassRule 34 | public static final EnvironmentVariables environmentVariables 35 | = new EnvironmentVariables(); 36 | 37 | @BeforeClass 38 | public static void fakeCredentialsDir() { 39 | environmentVariables.set("CREDENTIALS_DIR", new File("src/main/test/tokens").getAbsolutePath()); 40 | } 41 | 42 | @Autowired 43 | private MockNakadiPublishingClient mockClient; 44 | 45 | @Autowired 46 | private EventTransmitter eventTransmitter; 47 | 48 | @Before 49 | @After 50 | public void cleanUpMock() { 51 | eventTransmitter.sendEvents(); 52 | mockClient.clearSentEvents(); 53 | } 54 | 55 | @Test 56 | public void shouldSuccessfullyStartAndSnapshotCanBeTriggered() throws InterruptedException { 57 | given().baseUri("http://localhost:" + localManagementPort).contentType("application/json") 58 | .when().post("/actuator/snapshot-event-creation/eventtype") 59 | .then().statusCode(204); 60 | 61 | // leave some time for the scheduler to run 62 | Thread.sleep(200); 63 | 64 | List events = mockClient.getSentEvents("eventtype"); 65 | assertThat(events, hasSize(2)); 66 | } 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/transmission/MockNakadiPublishingClientTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission; 2 | 3 | 4 | import static java.util.Arrays.asList; 5 | import static java.util.Collections.singletonList; 6 | import lombok.AllArgsConstructor; 7 | import lombok.Getter; 8 | import lombok.NoArgsConstructor; 9 | import lombok.Setter; 10 | import static org.hamcrest.MatcherAssert.assertThat; 11 | import static org.hamcrest.Matchers.contains; 12 | import static org.hamcrest.Matchers.empty; 13 | import static org.hamcrest.Matchers.is; 14 | 15 | import org.junit.jupiter.api.Test; 16 | 17 | public class MockNakadiPublishingClientTest { 18 | 19 | private static final String MY_EVENT_TYPE = "my event type"; 20 | private static final String OTHER_EVENT_TYPE = "other event type"; 21 | 22 | private MockNakadiPublishingClient mockNakadiPublishingClient = new MockNakadiPublishingClient(); 23 | 24 | @Test 25 | public void returnsEmptyResultIfNoEventsHaveBeenSent() { 26 | assertThat(mockNakadiPublishingClient.getSentEvents("myEventType"), is(empty())); 27 | } 28 | 29 | @Test 30 | public void returnsOnlyThoseEventsOfTheGivenType() throws Exception { 31 | mockNakadiPublishingClient.publish(MY_EVENT_TYPE, singletonList(new Event("anEvent"))); 32 | mockNakadiPublishingClient.publish(OTHER_EVENT_TYPE, singletonList(new Event("anotherEvent"))); 33 | 34 | assertThat(mockNakadiPublishingClient.getSentEvents(MY_EVENT_TYPE), contains("{\"attribute\":\"anEvent\"}")); 35 | } 36 | 37 | @Test 38 | public void concatenatesSubsequentlyPublishedEventLists() throws Exception { 39 | mockNakadiPublishingClient.publish(MY_EVENT_TYPE, 40 | asList(new Event("event1"), new Event("event2")) 41 | ); 42 | mockNakadiPublishingClient.publish(MY_EVENT_TYPE, 43 | asList(new Event("event3"), new Event("event4")) 44 | ); 45 | 46 | assertThat( 47 | mockNakadiPublishingClient.getSentEvents(MY_EVENT_TYPE), 48 | contains( 49 | "{\"attribute\":\"event1\"}", 50 | "{\"attribute\":\"event2\"}", 51 | "{\"attribute\":\"event3\"}", 52 | "{\"attribute\":\"event4\"}" 53 | ) 54 | ); 55 | } 56 | 57 | @Test 58 | public void deletesAllEventsOnClear() throws Exception { 59 | mockNakadiPublishingClient.publish(MY_EVENT_TYPE, singletonList(new Event("event1"))); 60 | mockNakadiPublishingClient.clearSentEvents(); 61 | 62 | assertThat(mockNakadiPublishingClient.getSentEvents(MY_EVENT_TYPE), is(empty())); 63 | } 64 | 65 | @NoArgsConstructor 66 | @AllArgsConstructor 67 | @Getter 68 | @Setter 69 | public static class Event { 70 | private String attribute; 71 | } 72 | 73 | } -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SimpleSnapshotEventGenerator.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | import org.springframework.lang.Nullable; 4 | 5 | import java.util.List; 6 | import java.util.function.BiFunction; 7 | import java.util.function.Function; 8 | 9 | /** 10 | * This is a simple implementation of the {@link SnapshotEventGenerator}. It is 11 | * meant to be used in a functional style. 12 | * 13 | * @see SnapshotEventGenerator 14 | * @deprecated Please use the {@link SnapshotEventGenerator#of} methods instead. 15 | */ 16 | @Deprecated 17 | public final class SimpleSnapshotEventGenerator implements SnapshotEventGenerator { 18 | private final String supportedEventType; 19 | private final BiFunction> getSnapshotFunction; 20 | 21 | /** 22 | * Creates a SnapshotEventGenerator for an event type, which doesn't support 23 | * filtering. 24 | * 25 | * @param supportedEventType 26 | * the eventType that this SnapShotEventProvider will support. 27 | * @param snapshotEventFactory 28 | * a snapshot event factory function conforming to the 29 | * specification of 30 | * {@link SnapshotEventGenerator#generateSnapshots(Object, String)} 31 | * (but without the filter parameter). Any filter provided by the 32 | * caller will be thrown away. 33 | */ 34 | public SimpleSnapshotEventGenerator(String supportedEventType, 35 | Function> snapshotEventFactory) { 36 | this.supportedEventType = supportedEventType; 37 | this.getSnapshotFunction = (id, filter) -> snapshotEventFactory.apply(id); 38 | } 39 | 40 | /** 41 | * Creates a SnapshotEventGenerator for an event type, with filtering 42 | * support. 43 | * 44 | * @param supportedEventType 45 | * the eventType that this SnapShotEventProvider will support. 46 | * @param snapshotEventFactory 47 | * a snapshot event factory function conforming to the 48 | * specification of 49 | * {@link SnapshotEventGenerator#generateSnapshots(Object, String)}. 50 | */ 51 | public SimpleSnapshotEventGenerator(String supportedEventType, 52 | BiFunction> snapshotEventFactory) { 53 | this.supportedEventType = supportedEventType; 54 | this.getSnapshotFunction = snapshotEventFactory; 55 | } 56 | 57 | @Override 58 | public List generateSnapshots(@Nullable Object withIdGreaterThan, String filter) { 59 | return getSnapshotFunction.apply(withIdGreaterThan, filter); 60 | } 61 | 62 | @Override 63 | public String getSupportedEventType() { 64 | return supportedEventType; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotCreationService.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots.impl; 2 | 3 | import static java.util.Collections.unmodifiableSet; 4 | import static java.util.function.Function.identity; 5 | import static java.util.stream.Collectors.*; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 12 | import org.zalando.nakadiproducer.snapshots.Snapshot; 13 | import org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator; 14 | import org.zalando.nakadiproducer.snapshots.UnknownEventTypeException; 15 | 16 | public class SnapshotCreationService { 17 | 18 | private final Map snapshotEventProviders; 19 | 20 | private final EventLogWriter eventLogWriter; 21 | 22 | /** 23 | * Creates the service. 24 | * 25 | * @param snapshotEventGenerators 26 | * the event generators. Each of them must have a different 27 | * supported event type. 28 | * @param eventLogWriter 29 | * The event log writer to which the newly generated snapshot 30 | * events are pushed. 31 | * @throws IllegalStateException if two event generators declare to be responsible for the same event type. 32 | */ 33 | public SnapshotCreationService(List snapshotEventGenerators, 34 | EventLogWriter eventLogWriter) { 35 | this.snapshotEventProviders = snapshotEventGenerators.stream() 36 | .collect(toMap(SnapshotEventGenerator::getSupportedEventType, identity())); 37 | this.eventLogWriter = eventLogWriter; 38 | } 39 | 40 | public void createSnapshotEvents(final String eventType, String filter) { 41 | final SnapshotEventGenerator snapshotEventGenerator = snapshotEventProviders.get(eventType); 42 | if (snapshotEventGenerator == null) { 43 | throw new UnknownEventTypeException(eventType); 44 | } 45 | 46 | Object lastProcessedId = null; 47 | do { 48 | final List snapshots = snapshotEventGenerator.generateSnapshots(lastProcessedId, filter); 49 | if (snapshots.isEmpty()) { 50 | break; 51 | } 52 | 53 | snapshots.stream() 54 | .collect(groupingBy(Snapshot::getDataType, mapping(Snapshot::getData, toList()))) 55 | .forEach((dataType, snapshotPartition) -> 56 | eventLogWriter.fireSnapshotEvents(eventType, dataType, snapshotPartition)); 57 | 58 | lastProcessedId = snapshots.get(snapshots.size()-1).getId(); 59 | } while (true); 60 | } 61 | 62 | public Set getSupportedEventTypes() { 63 | return unmodifiableSet(snapshotEventProviders.keySet()); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/SubmissionDisabledIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.context.ApplicationContext; 7 | import org.springframework.test.context.ActiveProfiles; 8 | import org.zalando.fahrschein.NakadiClient; 9 | import org.zalando.fahrschein.http.api.RequestFactory; 10 | import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; 11 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 12 | import org.zalando.nakadiproducer.eventlog.impl.EventLogRepository; 13 | import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; 14 | import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; 15 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 16 | 17 | import static org.hamcrest.CoreMatchers.notNullValue; 18 | import static org.hamcrest.CoreMatchers.nullValue; 19 | import static org.hamcrest.MatcherAssert.assertThat; 20 | 21 | // no "test" profile, as this would include the mock client. 22 | @SpringBootTest( 23 | webEnvironment = SpringBootTest.WebEnvironment.MOCK, 24 | properties = {"nakadi-producer.submission-enabled:false"}, 25 | classes = { TestApplication.class, EmbeddedDataSourceConfig.class } 26 | ) 27 | public class SubmissionDisabledIT { 28 | 29 | @Autowired 30 | ApplicationContext context; 31 | 32 | @Test 33 | public void noNakadiBeans() { 34 | assertThat(context.getBeanProvider(NakadiClient.class).getIfAvailable(), nullValue()); 35 | assertThat(context.getBeanProvider(NakadiPublishingClient.class).getIfAvailable(), nullValue()); 36 | assertThat(context.getBeanProvider(StupsTokenComponent.class).getIfAvailable(), nullValue()); 37 | assertThat(context.getBeanProvider(RequestFactory.class).getIfAvailable(), nullValue()); 38 | } 39 | 40 | @Test 41 | public void noTransmissionBeans() { 42 | assertThat(context.getBeanProvider(EventTransmitter.class).getIfAvailable(), nullValue()); 43 | assertThat(context.getBeanProvider(EventTransmissionService.class).getIfAvailable(), nullValue()); 44 | assertThat(context.getBeanProvider(EventTransmissionScheduler.class).getIfAvailable(), nullValue()); 45 | } 46 | 47 | @Test 48 | public void yesEventLogWriter() { 49 | assertThat(context.getBeanProvider(EventLogWriter.class).getIfAvailable(), notNullValue()); 50 | } 51 | 52 | @Test 53 | public void yesRepository() { 54 | assertThat(context.getBeanProvider(EventLogRepository.class).getIfAvailable(), notNullValue()); 55 | } 56 | 57 | @Test 58 | public void yesFlywayMigrator() { 59 | assertThat(context.getBeanProvider(FlywayMigrator.class).getIfAvailable(), notNullValue()); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | 3.2 8 | 9 | 10 | 11 | org.springframework.boot 12 | spring-boot-starter-parent 13 | 2.5.6 14 | 15 | 16 | nakadi-producer-reactor 17 | org.zalando 18 | 21.0.0 19 | pom 20 | Nakadi Event Producer Reactor 21 | 22 | 23 | nakadi-producer 24 | nakadi-producer-spring-boot-starter 25 | nakadi-producer-starter-spring-boot-2-test 26 | nakadi-producer-loadtest 27 | 28 | 29 | 30 | 31 | MIT 32 | https://opensource.org/licenses/MIT 33 | repo 34 | 35 | 36 | 37 | 38 | 39 | ${project.artifactId} 40 | 41 | 42 | 43 | org.apache.maven.plugins 44 | maven-gpg-plugin 45 | 1.5 46 | 47 | 48 | sign-artifacts 49 | verify 50 | 51 | sign 52 | 53 | 54 | 55 | 56 | 57 | org.sonatype.plugins 58 | nexus-staging-maven-plugin 59 | 1.6.7 60 | true 61 | 62 | ossrh 63 | https://oss.sonatype.org/ 64 | true 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | ossrh 73 | https://oss.sonatype.org/content/repositories/snapshots 74 | 75 | 76 | ossrh 77 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/EventBatcher.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import lombok.AllArgsConstructor; 5 | import lombok.EqualsAndHashCode; 6 | import lombok.Getter; 7 | import lombok.ToString; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.zalando.nakadiproducer.eventlog.impl.EventLog; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.function.Consumer; 14 | 15 | @Slf4j 16 | public class EventBatcher { 17 | 18 | private static final long NAKADI_BATCH_SIZE_LIMIT_IN_BYTES = 50000000; 19 | private final ObjectMapper objectMapper; 20 | private final Consumer> publisher; 21 | 22 | private List batch; 23 | private long aggregatedBatchSize; 24 | 25 | public EventBatcher(ObjectMapper objectMapper, Consumer> publisher) { 26 | this.objectMapper = objectMapper; 27 | this.publisher = publisher; 28 | 29 | this.batch = new ArrayList<>(); 30 | this.aggregatedBatchSize = 0; 31 | } 32 | 33 | /** 34 | * Pushes one event to be published. It will be either published right now, or with some other events, 35 | * latest when calling {@link #finish()}. 36 | * @param eventLogEntry The event log entry for this event. 37 | * @param nakadiEvent The Nakadi form of the event. 38 | */ 39 | public void pushEvent(EventLog eventLogEntry, NakadiEvent nakadiEvent) { 40 | long eventSize; 41 | 42 | try { 43 | eventSize = objectMapper.writeValueAsBytes(nakadiEvent).length; 44 | } catch (Exception e) { 45 | log.error("Could not serialize event {} of type {}, skipping it.", eventLogEntry.getId(), eventLogEntry.getEventType(), e); 46 | return; 47 | } 48 | 49 | 50 | if (!batch.isEmpty() && 51 | (hasAnotherEventType(batch, eventLogEntry) || batchWouldBecomeTooBig(aggregatedBatchSize, eventSize))) { 52 | this.publisher.accept(batch); 53 | 54 | batch = new ArrayList<>(); 55 | aggregatedBatchSize = 0; 56 | } 57 | 58 | batch.add(new BatchItem(eventLogEntry, nakadiEvent)); 59 | aggregatedBatchSize += eventSize; 60 | } 61 | 62 | /** 63 | * Publishes all events which were pushed and not yet published. 64 | */ 65 | public void finish() { 66 | if (!batch.isEmpty()) { 67 | this.publisher.accept(batch); 68 | } 69 | } 70 | 71 | private boolean hasAnotherEventType(List batch, EventLog event) { 72 | return !event.getEventType().equals(batch.get(0).getEventLogEntry().getEventType()); 73 | } 74 | 75 | private boolean batchWouldBecomeTooBig(long aggregatedBatchSize, long eventSize) { 76 | return aggregatedBatchSize + eventSize > 0.8 * NAKADI_BATCH_SIZE_LIMIT_IN_BYTES; 77 | } 78 | 79 | @AllArgsConstructor 80 | @Getter 81 | @EqualsAndHashCode 82 | @ToString 83 | protected static class BatchItem { 84 | EventLog eventLogEntry; 85 | NakadiEvent nakadiEvent; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockingIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.junit.jupiter.api.AfterEach; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 8 | import org.zalando.nakadiproducer.eventlog.impl.EventLog; 9 | import org.zalando.nakadiproducer.eventlog.impl.EventLogRepository; 10 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 11 | import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; 12 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 13 | import org.zalando.nakadiproducer.util.Fixture; 14 | 15 | import java.time.Clock; 16 | import java.time.Instant; 17 | import java.time.ZoneId; 18 | import java.util.Collection; 19 | import java.util.List; 20 | 21 | import static java.time.temporal.ChronoUnit.MINUTES; 22 | import static org.hamcrest.MatcherAssert.assertThat; 23 | import static org.hamcrest.Matchers.is; 24 | 25 | public class LockingIT extends BaseMockedExternalCommunicationIT { 26 | private static final String MY_EVENT_TYPE = "myEventType"; 27 | 28 | @Autowired 29 | private EventLogWriter eventLogWriter; 30 | 31 | @Autowired 32 | private EventTransmitter eventTransmitter; 33 | 34 | @Autowired 35 | private EventLogRepository eventLogRepository; 36 | 37 | @Autowired 38 | private EventTransmissionService eventTransmissionService; 39 | 40 | @Autowired 41 | private MockNakadiPublishingClient nakadiClient; 42 | 43 | @BeforeEach 44 | @AfterEach 45 | public void clearNakadiEvents() { 46 | eventTransmitter.sendEvents(); 47 | nakadiClient.clearSentEvents(); 48 | eventLogRepository.deleteAll(); 49 | } 50 | 51 | @Test 52 | public void eventsShouldNotBeSentTwiceWhenLockExpiresDuringTransmission() { 53 | // Given that there is an event to be sent... 54 | eventLogWriter.fireBusinessEvent(MY_EVENT_TYPE, Fixture.mockPayload(1, "code123")); 55 | 56 | // ... and given one job instance locked it for sending... 57 | Instant timeOfInitialLock = Instant.now(); 58 | mockServiceClock(timeOfInitialLock); 59 | Collection eventLogsLockedFirst = eventTransmissionService.lockSomeEvents(); 60 | 61 | // ... and given that so much time passed in the meantime that the lock already expired... 62 | mockServiceClock(timeOfInitialLock.plus(11, MINUTES)); 63 | 64 | // ... so that another job could have locked the same events ... 65 | Collection eventLogsLockedSecond = eventTransmissionService.lockSomeEvents(); 66 | 67 | // when both job instances try to send their locked events 68 | eventTransmissionService.sendEvents(eventLogsLockedFirst); 69 | eventTransmissionService.sendEvents(eventLogsLockedSecond); 70 | 71 | // Then the event should have been sent only once. 72 | List value = nakadiClient.getSentEvents(MY_EVENT_TYPE); 73 | 74 | assertThat(value.size(), is(1)); 75 | } 76 | 77 | private void mockServiceClock(Instant ins) { 78 | eventTransmissionService.overrideClock(Clock.fixed(ins, ZoneId.systemDefault())); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the [project maintainers][maintainers]. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | [maintainers]: https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/blob/master/MAINTAINERS -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockTimeoutIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.junit.jupiter.api.AfterEach; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.test.context.SpringBootTest; 8 | import org.springframework.transaction.annotation.Transactional; 9 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 10 | import org.zalando.nakadiproducer.eventlog.impl.EventLog; 11 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 12 | import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; 13 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 14 | import org.zalando.nakadiproducer.util.Fixture; 15 | 16 | import java.time.Clock; 17 | import java.time.Instant; 18 | import java.time.ZoneId; 19 | import java.util.Collection; 20 | 21 | import static java.time.temporal.ChronoUnit.SECONDS; 22 | import static org.hamcrest.MatcherAssert.assertThat; 23 | import static org.hamcrest.Matchers.empty; 24 | import static org.hamcrest.Matchers.is; 25 | 26 | @Transactional 27 | @SpringBootTest(properties = { 28 | "nakadi-producer.scheduled-transmission-enabled:false", 29 | "nakadi-producer.lock-duration:300", 30 | "nakadi-producer.lock-duration-buffer:30"}) 31 | public class LockTimeoutIT extends BaseMockedExternalCommunicationIT { 32 | private static final String MY_EVENT_TYPE = "myEventType"; 33 | 34 | @Autowired 35 | private EventLogWriter eventLogWriter; 36 | 37 | @Autowired 38 | private EventTransmitter eventTransmitter; 39 | 40 | @Autowired 41 | private EventTransmissionService eventTransmissionService; 42 | 43 | @Autowired 44 | private MockNakadiPublishingClient nakadiClient; 45 | 46 | @BeforeEach 47 | @AfterEach 48 | public void clearNakadiEvents() { 49 | mockServiceClock(Instant.now()); 50 | eventTransmitter.sendEvents(); 51 | nakadiClient.clearSentEvents(); 52 | } 53 | 54 | @Test 55 | public void testLockedUntil() { 56 | eventLogWriter.fireBusinessEvent(MY_EVENT_TYPE, Fixture.mockPayload(1, "code123")); 57 | 58 | Instant timeOfInitialLock = Instant.now(); 59 | mockServiceClock(timeOfInitialLock); 60 | 61 | assertThat(eventTransmissionService.lockSomeEvents().size(), is(1)); 62 | assertThat(eventTransmissionService.lockSomeEvents(), empty()); 63 | 64 | // lock is still valid 65 | mockServiceClock(timeOfInitialLock.plus(300 - 5, SECONDS)); 66 | assertThat(eventTransmissionService.lockSomeEvents(), empty()); 67 | 68 | // lock is expired 69 | mockServiceClock(timeOfInitialLock.plus(300 + 5, SECONDS)); 70 | assertThat(eventTransmissionService.lockSomeEvents().size(), is(1)); 71 | } 72 | 73 | @Test 74 | public void testLockNearlyExpired() { 75 | eventLogWriter.fireBusinessEvent(MY_EVENT_TYPE, Fixture.mockPayload(1, "code456")); 76 | Instant timeOfInitialLock = Instant.now(); 77 | 78 | Collection lockedEvent = eventTransmissionService.lockSomeEvents(); 79 | 80 | // event will not be sent, because the event-lock is "nearlyExpired" 81 | mockServiceClock(timeOfInitialLock.plus(300 - 30 + 5, SECONDS)); 82 | eventTransmissionService.sendEvents(lockedEvent); 83 | assertThat(nakadiClient.getSentEvents(MY_EVENT_TYPE), empty()); 84 | } 85 | 86 | private void mockServiceClock(Instant ins) { 87 | eventTransmissionService.overrideClock(Clock.fixed(ins, ZoneId.systemDefault())); 88 | } 89 | } -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | import static com.jayway.restassured.RestAssured.given; 4 | import static org.mockito.Mockito.mock; 5 | import static org.mockito.Mockito.reset; 6 | import static org.mockito.Mockito.verify; 7 | import static org.mockito.Mockito.when; 8 | 9 | import org.junit.jupiter.api.BeforeEach; 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.boot.actuate.autoconfigure.web.server.LocalManagementPort; 13 | import org.springframework.boot.test.context.SpringBootTest; 14 | import org.springframework.context.annotation.Bean; 15 | import org.springframework.context.annotation.Configuration; 16 | import org.springframework.test.context.ActiveProfiles; 17 | import org.zalando.nakadiproducer.TestApplication; 18 | import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; 19 | 20 | @ActiveProfiles("test") 21 | @SpringBootTest( 22 | webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, 23 | properties = { 24 | "management.security.enabled=false", 25 | "zalando.team.id:alpha-local-testing", 26 | "nakadi-producer.scheduled-transmission-enabled:false", 27 | "management.endpoints.web.exposure.include:snapshot-event-creation" 28 | }, 29 | classes = {TestApplication.class, EmbeddedDataSourceConfig.class, SnapshotEventGenerationWebEndpointIT.Config.class} 30 | ) 31 | public class SnapshotEventGenerationWebEndpointIT { 32 | 33 | private static final String MY_EVENT_TYPE = "my.event-type"; 34 | private static final String FILTER = "myRequestBody"; 35 | 36 | @LocalManagementPort 37 | private int managementPort; 38 | 39 | @Autowired 40 | private SnapshotEventGenerator snapshotEventGenerator; 41 | 42 | @BeforeEach 43 | public void resetMocks() { 44 | reset(snapshotEventGenerator); 45 | } 46 | 47 | @Test 48 | public void passesFilterIfPresentInUrl() { 49 | given().baseUri("http://localhost:" + managementPort) 50 | .contentType("application/json") 51 | .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE + "?filter=" + FILTER) 52 | .then().statusCode(204); 53 | 54 | verify(snapshotEventGenerator).generateSnapshots(null, FILTER); 55 | } 56 | 57 | @Test 58 | public void passesFilterIfPresentInBody() { 59 | given().baseUri("http://localhost:" + managementPort) 60 | .contentType("application/json") 61 | .body("{\"filter\":\"" + FILTER + "\"}") 62 | .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE) 63 | .then().statusCode(204); 64 | 65 | verify(snapshotEventGenerator).generateSnapshots(null, FILTER); 66 | } 67 | 68 | @Test 69 | public void passesNullIfNoFilterIsPresent() { 70 | given().baseUri("http://localhost:" + managementPort) 71 | .contentType("application/json") 72 | .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE) 73 | .then().statusCode(204); 74 | 75 | verify(snapshotEventGenerator).generateSnapshots(null, null); 76 | } 77 | 78 | @Configuration 79 | public static class Config { 80 | @Bean 81 | public SnapshotEventGenerator snapshotEventGenerator() { 82 | SnapshotEventGenerator mock = mock(SnapshotEventGenerator.class); 83 | when(mock.getSupportedEventType()).thenReturn(MY_EVENT_TYPE); 84 | return mock; 85 | } 86 | 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/eventlog/CompactionKeyExtractor.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog; 2 | 3 | import org.zalando.nakadiproducer.eventlog.CompactionKeyExtractors.SimpleCompactionKeyExtractor; 4 | import org.zalando.nakadiproducer.eventlog.CompactionKeyExtractors.TypedCompactionKeyExtractor; 5 | 6 | import java.util.Optional; 7 | import java.util.function.Function; 8 | 9 | /** 10 | * This interface defines a way of extracting a compaction key from an object which 11 | * is sent as a payload in a compacted event type. 12 | * In most cases, for each compacted event type exactly one such object will be made known to the producer, and 13 | * you can define it using {@link #of(String, Class, Function)}, passing a method reference or a lambda. 14 | * For special occasions (e.g. where objects of different classes are used as payloads for the same event type) 15 | * also multiple extractors for the same event type are supported – in this case any which returns a 16 | * non-empty optional will be used. 17 | */ 18 | public interface CompactionKeyExtractor { 19 | 20 | default String getKeyOrNull(Object payload) { 21 | return tryGetKeyFor(payload).orElse(null); 22 | } 23 | 24 | Optional tryGetKeyFor(Object o); 25 | 26 | String getEventType(); 27 | 28 | /** 29 | * A type-safe compaction key extractor. This will be the one to be used by most applications. 30 | * 31 | * @param eventType Indicates the event type. Only events sent to this event type will be considered. 32 | * @param type A Java type for payload objects. Only payload objects where {@code type.isInstance(payload)} 33 | * will be considered at all. 34 | * @param extractorFunction A function extracting a compaction key from a payload object. 35 | * This will commonly be given as a method reference or lambda. 36 | * @return A compaction key extractor, to be defined as a spring bean (if using the spring-boot starter) 37 | * or passed manually to the event log writer implementation (if using nakadi-producer directly). 38 | * (This should not return null.) 39 | * @param the type of {@code type} and input type of {@code extractorFunction}. 40 | */ 41 | static CompactionKeyExtractor of(String eventType, Class type, Function extractorFunction) { 42 | return new TypedCompactionKeyExtractor<>(eventType, type, extractorFunction); 43 | } 44 | 45 | /** 46 | * Non-type safe key extractor, returning an Optional. 47 | * @param eventType The event type for which this extractor is intended. 48 | * @param extractor The extractor function. It is supposed to return {@link Optional#empty()} if this extractor 49 | * can't handle the input object, otherwise the actual key. 50 | * @return a key extractor object. 51 | */ 52 | static CompactionKeyExtractor ofOptional(String eventType, Function> extractor) { 53 | return new SimpleCompactionKeyExtractor(eventType, extractor); 54 | } 55 | 56 | /** 57 | * Non-type safe key extractor, returning null for unknown objects. 58 | * @param eventType The event type for which this extractor is intended. 59 | * @param extractor The extractor function. It is supposed to return {@code null} if this extractor 60 | * can't handle the input object, otherwise the actual key. 61 | * @return a key extractor object. 62 | */ 63 | static CompactionKeyExtractor ofNullable(String eventType, Function extractor) { 64 | return new SimpleCompactionKeyExtractor(eventType, extractor.andThen(Optional::ofNullable)); 65 | } 66 | 67 | /** 68 | * An universal key extractor, capable of handling all objects. 69 | * @param eventType The event type for which this extractor is intended. 70 | * @param extractor The extractor function. It is not allowed to return {@code null}. 71 | * @return a key extractor object. 72 | */ 73 | static CompactionKeyExtractor of(String eventType, Function extractor) { 74 | return new SimpleCompactionKeyExtractor(eventType, extractor.andThen(Optional::of)); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/eventlog/impl/EventLogRepositoryIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import static org.hamcrest.MatcherAssert.assertThat; 4 | import static org.hamcrest.Matchers.hasSize; 5 | import static org.hamcrest.core.Is.is; 6 | 7 | import javax.transaction.Transactional; 8 | 9 | import org.junit.jupiter.api.BeforeEach; 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.jdbc.core.BeanPropertyRowMapper; 13 | import org.springframework.jdbc.core.JdbcTemplate; 14 | import org.zalando.nakadiproducer.BaseMockedExternalCommunicationIT; 15 | 16 | import java.util.List; 17 | 18 | @Transactional 19 | public class EventLogRepositoryIT extends BaseMockedExternalCommunicationIT { 20 | 21 | @Autowired 22 | private EventLogRepository eventLogRepository; 23 | 24 | @Autowired 25 | private JdbcTemplate jdbcTemplate; 26 | 27 | private static final String WAREHOUSE_EVENT_BODY_DATA = 28 | ("{'self':'http://WAREHOUSE_DOMAIN'," 29 | + "'code':'WH-DE-EF'," 30 | + "'name':'Erfurt'," 31 | + "'address':{'name':'Zalando Logistics SE & Co.KG'," 32 | + "'street':'In der Hochstedter Ecke 1'," 33 | + "'city':'Erfurt'," 34 | + "'zip':'99098'," 35 | + "'country':'DE'," 36 | + "'additional':null" 37 | + "}," 38 | + "'is_allowed_for_shipping':true," 39 | + "'is_allowed_for_purchase_order':true," 40 | + "'legacy_warehouse_code':'3'" 41 | + "}").replace('\'', '"'); 42 | 43 | private final String WAREHOUSE_EVENT_TYPE = "wholesale.warehouse-change-event"; 44 | 45 | public static final String COMPACTION_KEY = "COMPACTED"; 46 | 47 | @BeforeEach 48 | public void setUp() { 49 | eventLogRepository.deleteAll(); 50 | 51 | persistTestEvent("FLOW_ID"); 52 | } 53 | 54 | private void persistTestEvent(String flowId) { 55 | final EventLog eventLog = EventLog.builder() 56 | .eventBodyData(WAREHOUSE_EVENT_BODY_DATA) 57 | .eventType(WAREHOUSE_EVENT_TYPE) 58 | .compactionKey(COMPACTION_KEY) 59 | .flowId(flowId) 60 | .build(); 61 | eventLogRepository.persist(eventLog); 62 | } 63 | 64 | @Test 65 | public void testDeleteMultipleEvents() { 66 | persistTestEvent("second_Flow-ID"); 67 | persistTestEvent("third flow-ID"); 68 | persistTestEvent("fourth flow-ID"); 69 | persistTestEvent("fifth flow-ID"); 70 | 71 | List events = findAllEventsInDB(); 72 | assertThat(events, hasSize(5)); 73 | EventLog notDeleted = events.remove(0); 74 | 75 | // now the actual test – delete just 4 of the 5 events from the DB 76 | eventLogRepository.delete(events); 77 | 78 | List remaining = findAllEventsInDB(); 79 | assertThat(remaining, hasSize(1)); 80 | assertThat(remaining.get(0).getId(), is(notDeleted.getId())); 81 | assertThat(remaining.get(0).getFlowId(), is(notDeleted.getFlowId())); 82 | } 83 | 84 | private List findAllEventsInDB() { 85 | return jdbcTemplate.query( 86 | "SELECT * FROM nakadi_events.event_log", 87 | new BeanPropertyRowMapper<>(EventLog.class)); 88 | } 89 | 90 | @Test 91 | public void testFindEventInRepositoryById() { 92 | Integer id = jdbcTemplate.queryForObject( 93 | "SELECT id FROM nakadi_events.event_log WHERE flow_id = 'FLOW_ID'", 94 | Integer.class); 95 | final EventLog eventLog = eventLogRepository.findOne(id); 96 | compareWithPersistedEvent(eventLog); 97 | } 98 | 99 | private void compareWithPersistedEvent(final EventLog eventLog) { 100 | assertThat(eventLog.getEventBodyData(), is(WAREHOUSE_EVENT_BODY_DATA)); 101 | assertThat(eventLog.getEventType(), is(WAREHOUSE_EVENT_TYPE)); 102 | assertThat(eventLog.getCompactionKey(), is(COMPACTION_KEY)); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotCreationServiceTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots.impl; 2 | 3 | import static java.util.Arrays.asList; 4 | import static java.util.Collections.emptyList; 5 | import static java.util.Collections.singletonList; 6 | import static java.util.stream.Collectors.toList; 7 | import static org.hamcrest.MatcherAssert.assertThat; 8 | import static org.hamcrest.Matchers.contains; 9 | import static org.hamcrest.Matchers.equalTo; 10 | import static org.hamcrest.core.Is.is; 11 | import static org.mockito.ArgumentMatchers.eq; 12 | import static org.mockito.Mockito.times; 13 | import static org.mockito.Mockito.verify; 14 | import static org.mockito.Mockito.when; 15 | import static org.zalando.nakadiproducer.util.Fixture.PUBLISHER_DATA_TYPE; 16 | import static org.zalando.nakadiproducer.util.Fixture.PUBLISHER_EVENT_TYPE; 17 | 18 | import java.util.Collection; 19 | import java.util.List; 20 | 21 | import org.junit.jupiter.api.BeforeEach; 22 | import org.junit.jupiter.api.Test; 23 | import org.junit.jupiter.api.extension.ExtendWith; 24 | import org.mockito.ArgumentCaptor; 25 | import org.mockito.Captor; 26 | import org.mockito.Mock; 27 | import org.mockito.junit.jupiter.MockitoExtension; 28 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 29 | import org.zalando.nakadiproducer.snapshots.Snapshot; 30 | import org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator; 31 | import org.zalando.nakadiproducer.util.Fixture; 32 | import org.zalando.nakadiproducer.util.MockPayload; 33 | 34 | @ExtendWith(MockitoExtension.class) 35 | public class SnapshotCreationServiceTest { 36 | 37 | @Mock 38 | private SnapshotEventGenerator snapshotEventGenerator; 39 | 40 | @Mock 41 | private EventLogWriter eventLogWriter; 42 | 43 | private SnapshotCreationService snapshotCreationService; 44 | 45 | @Captor 46 | private ArgumentCaptor> eventLogDataCaptor; 47 | 48 | @BeforeEach 49 | public void setUp() throws Exception { 50 | when(snapshotEventGenerator.getSupportedEventType()).thenReturn(PUBLISHER_EVENT_TYPE); 51 | snapshotCreationService = new SnapshotCreationService(asList(snapshotEventGenerator), eventLogWriter); 52 | } 53 | 54 | @Test 55 | public void testCreateSnapshotEvents() { 56 | final String filter = "exampleFilter"; 57 | 58 | MockPayload eventPayload = Fixture.mockPayload(1, "mockedcode", true, 59 | Fixture.mockSubClass("some info"), Fixture.mockSubList(2, "some detail")); 60 | 61 | when(snapshotEventGenerator.generateSnapshots(null, filter)).thenReturn( 62 | singletonList(new Snapshot(1, PUBLISHER_DATA_TYPE, eventPayload))); 63 | 64 | snapshotCreationService.createSnapshotEvents(PUBLISHER_EVENT_TYPE, filter); 65 | 66 | verify(eventLogWriter).fireSnapshotEvents(eq(PUBLISHER_EVENT_TYPE), eq(PUBLISHER_DATA_TYPE), 67 | eventLogDataCaptor.capture()); 68 | assertThat(eventLogDataCaptor.getValue(), contains(eventPayload)); 69 | } 70 | 71 | @Test 72 | public void testSnapshotSavedInBatches() { 73 | final String filter = "exampleFilter2"; 74 | 75 | final List eventSnapshots = Fixture.mockSnapshotList(5); 76 | 77 | // when snapshot returns 5 item stream 78 | when(snapshotEventGenerator.generateSnapshots(null, filter)).thenReturn(eventSnapshots.subList(0, 3)); 79 | when(snapshotEventGenerator.generateSnapshots(2, filter)).thenReturn(eventSnapshots.subList(3, 5)); 80 | when(snapshotEventGenerator.generateSnapshots(4, filter)).thenReturn(emptyList()); 81 | 82 | // create a snapshot 83 | snapshotCreationService.createSnapshotEvents(PUBLISHER_EVENT_TYPE, filter); 84 | 85 | // verify that all returned events got written 86 | verify(eventLogWriter, times(2)).fireSnapshotEvents(eq(PUBLISHER_EVENT_TYPE), eq(PUBLISHER_DATA_TYPE), 87 | eventLogDataCaptor.capture()); 88 | 89 | List payloads = eventSnapshots.stream().map(Snapshot::getData).collect(toList()); 90 | List writtenEvents = eventLogDataCaptor 91 | .getAllValues() 92 | .stream() 93 | .flatMap(Collection::stream) 94 | .collect(toList()); 95 | assertThat(writtenEvents, is(equalTo(payloads))); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /nakadi-producer-starter-spring-boot-2-test/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | nakadi-producer-starter-spring-boot-2-test 7 | org.zalando 8 | 9 | 10 | 11 | org.zalando 12 | nakadi-producer-reactor 13 | 21.0.0 14 | 15 | 16 | 17 | 18 | org.zalando 19 | nakadi-producer-spring-boot-starter 20 | ${project.version} 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-web 25 | 26 | 27 | org.zalando.stups 28 | tokens 29 | 0.14.0 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-jdbc 34 | 2.5.6 35 | 36 | 37 | com.opentable.components 38 | otj-pg-embedded 39 | 0.12.0 40 | 41 | 42 | postgresql 43 | postgresql 44 | 45 | 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-actuator 50 | 51 | 52 | org.springframework.boot 53 | spring-boot-starter-test 54 | test 55 | 56 | 57 | com.github.stefanbirkner 58 | system-rules 59 | 1.19.0 60 | test 61 | 62 | 63 | junit 64 | junit-dep 65 | 66 | 67 | 68 | 70 | 71 | junit 72 | junit-dep 73 | 4.11 74 | test 75 | 76 | 77 | io.rest-assured 78 | rest-assured 79 | 4.4.0 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | org.apache.maven.plugins 88 | maven-failsafe-plugin 89 | 2.22.0 90 | 91 | 92 | 93 | integration-test 94 | verify 95 | 96 | 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-deploy-plugin 102 | 103 | true 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerator.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.snapshots; 2 | 3 | 4 | import org.springframework.lang.Nullable; 5 | 6 | import java.util.List; 7 | import java.util.function.BiFunction; 8 | import java.util.function.Function; 9 | 10 | /** 11 | * The {@code SnapshotEventGenerator} interface should be implemented by any 12 | * event producer that wants to support the snapshot events feature. The class 13 | * must define a method {@link #generateSnapshots}, as well as 14 | * {@link #getSupportedEventType()}. 15 | * 16 | * The {@link #of} methods can be used for creating an implementation from method references or lambdas: 17 | * 
{@code
 18 |  *  @Bean
 19 |  *  public SnapshotEventGenerator snapshotEventGenerator(MyService service) {
 20 |  *     return SnapshotEventGenerator.of("event type", service::createSnapshotEvents);
 21 |  *  }
 22 |  * }
23 | */ 24 | public interface SnapshotEventGenerator { 25 | 26 | /** 27 | *

28 | * Returns a batch of snapshots of given type (event type is an event 29 | * channel topic name). The implementation may return an arbitrary amount of 30 | * results, but it must return at least one element if there are entities 31 | * matching the parameters. 32 | *

33 | *

34 | * Calling this method must have no side effects. 35 | *

36 | * The library will call your implementation like this: 37 | *
    38 | *
  • Request: generateSnapshots(null, filter), Response: 1,2,3
    • 39 | *
  • Request: generateSnapshots(3, filter), Response: 4,5
    • 40 | *
  • Request: generateSnapshots(5, filter), Response: emptyList
    • 41 | *
42 | *

43 | * It is your responsibility to make sure that the returned events are 44 | * ordered by their ID ascending and that, given you return a list of events 45 | * for entities with IDs {id1, ..., idN}, there exists 46 | * no entity with an ID between id1 and idN, that is 47 | * not part of the result. 48 | *

49 | * 50 | * @param withIdGreaterThan 51 | * if not null, only events for entities with an ID greater than 52 | * the given one must be returned 53 | * 54 | * @param filter 55 | * a filter for the snapshot generation mechanism. This value is 56 | * simply passed through from the request body of the REST 57 | * endpoint (or from any other triggering mechanism). If there 58 | * was no request body, this will be {@code null}. 59 | * 60 | * Implementors can interpret it in whatever way they want (even 61 | * ignore it). All calls for one snapshot generation will receive 62 | * the same string. 63 | * 64 | * @return list of elements (wrapped in Snapshot objects) ordered by their 65 | * ID. 66 | */ 67 | List generateSnapshots(@Nullable Object withIdGreaterThan, String filter); 68 | 69 | /** 70 | * The name of the event type supported by this snapshot generator. 71 | */ 72 | String getSupportedEventType(); 73 | 74 | /** 75 | * Creates a SnapshotEventGenerator for an event type, which doesn't support 76 | * filtering. 77 | * 78 | * @param eventType 79 | * the eventType that this SnapShotEventProvider will support. 80 | * @param generator 81 | * a snapshot event factory function conforming to the 82 | * specification of 83 | * {@link SnapshotEventGenerator#generateSnapshots(Object, String)} 84 | * (but without the filter parameter). Any filter provided by the 85 | * caller will be thrown away. 86 | * @since 21.1.0 87 | */ 88 | static SnapshotEventGenerator of(String eventType, Function> generator) { 89 | return new SimpleSnapshotEventGenerator(eventType, generator); 90 | } 91 | 92 | /** 93 | * Creates a SnapshotEventGenerator for an event type, with filtering 94 | * support. 95 | * 96 | * @param eventType 97 | * the eventType that this SnapShotEventProvider will support. 98 | * @param generator 99 | * a snapshot event factory function conforming to the 100 | * specification of 101 | * {@link SnapshotEventGenerator#generateSnapshots(Object, String)}. 102 | * @since 21.1.0 103 | */ 104 | static SnapshotEventGenerator of(String eventType, BiFunction> generator) { 105 | return new SimpleSnapshotEventGenerator(eventType, generator); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import org.flywaydb.core.api.MigrationInfo; 4 | 5 | import java.sql.Connection; 6 | 7 | /** 8 | * This is the main callback interface that should be implemented to get access to flyway lifecycle notifications. 9 | * Simply add code to the callback method you are interested in having. 10 | * 11 | *

Each callback method will run within its own transaction.

12 | */ 13 | public interface NakadiProducerFlywayCallback { 14 | /** 15 | * Runs before the clean task executes. 16 | * 17 | * @param connection A valid connection to the database. 18 | */ 19 | default void beforeClean(Connection connection) {} 20 | 21 | /** 22 | * Runs after the clean task executes. 23 | * 24 | * @param connection A valid connection to the database. 25 | */ 26 | default void afterClean(Connection connection) {}; 27 | 28 | /** 29 | * Runs before the migrate task executes. 30 | * 31 | * @param connection A valid connection to the database. 32 | */ 33 | default void beforeMigrate(Connection connection) {}; 34 | 35 | /** 36 | * Runs after the migrate task executes. 37 | * 38 | * @param connection A valid connection to the database. 39 | */ 40 | default void afterMigrate(Connection connection) {} 41 | 42 | /** 43 | * Runs before the undo task executes. 44 | * 45 | * @param connection A valid connection to the database. 46 | */ 47 | default void beforeUndo(Connection connection) {} 48 | 49 | /** 50 | * Runs before each migration script is undone. 51 | * 52 | * @param connection A valid connection to the database. 53 | * @param info The current MigrationInfo for the migration to be undone. 54 | */ 55 | default void beforeEachUndo(Connection connection, MigrationInfo info) {} 56 | 57 | /** 58 | * Runs after each migration script is undone. 59 | * 60 | * @param connection A valid connection to the database. 61 | * @param info The current MigrationInfo for the migration just undone. 62 | */ 63 | default void afterEachUndo(Connection connection, MigrationInfo info) {} 64 | 65 | /** 66 | * Runs after the undo task executes. 67 | * 68 | * @param connection A valid connection to the database. 69 | */ 70 | default void afterUndo(Connection connection) {} 71 | 72 | /** 73 | * Runs before each migration script is executed. 74 | * 75 | * @param connection A valid connection to the database. 76 | * @param info The current MigrationInfo for this migration. 77 | */ 78 | default void beforeEachMigrate(Connection connection, MigrationInfo info) {} 79 | 80 | /** 81 | * Runs after each migration script is executed. 82 | * 83 | * @param connection A valid connection to the database. 84 | * @param info The current MigrationInfo for this migration. 85 | */ 86 | default void afterEachMigrate(Connection connection, MigrationInfo info) {} 87 | 88 | /** 89 | * Runs before the validate task executes. 90 | * 91 | * @param connection A valid connection to the database. 92 | */ 93 | default void beforeValidate(Connection connection) {} 94 | 95 | /** 96 | * Runs after the validate task executes. 97 | * 98 | * @param connection A valid connection to the database. 99 | */ 100 | default void afterValidate(Connection connection) {} 101 | 102 | /** 103 | * Runs before the baseline task executes. 104 | * 105 | * @param connection A valid connection to the database. 106 | */ 107 | default void beforeBaseline(Connection connection) {} 108 | 109 | /** 110 | * Runs after the baseline task executes. 111 | * 112 | * @param connection A valid connection to the database. 113 | */ 114 | default void afterBaseline(Connection connection) {} 115 | 116 | /** 117 | * Runs before the repair task executes. 118 | * 119 | * @param connection A valid connection to the database. 120 | */ 121 | default void beforeRepair(Connection connection) {} 122 | 123 | /** 124 | * Runs after the repair task executes. 125 | * 126 | * @param connection A valid connection to the database. 127 | */ 128 | default void afterRepair(Connection connection) {} 129 | 130 | /** 131 | * Runs before the info task executes. 132 | * 133 | * @param connection A valid connection to the database. 134 | */ 135 | default void beforeInfo(Connection connection) {} 136 | 137 | /** 138 | * Runs after the info task executes. 139 | * 140 | * @param connection A valid connection to the database. 141 | */ 142 | default void afterInfo(Connection connection) {} 143 | } 144 | -------------------------------------------------------------------------------- /nakadi-producer-loadtest/src/test/java/org/zalando/nakadiproducer/LoadTestIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.junit.After; 5 | import org.junit.Before; 6 | import org.junit.ClassRule; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.boot.test.context.SpringBootTest; 11 | import org.springframework.http.*; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.TestPropertySource; 14 | import org.springframework.test.context.junit4.SpringRunner; 15 | import org.springframework.web.client.RestTemplate; 16 | import org.testcontainers.containers.DockerComposeContainer; 17 | import org.testcontainers.containers.wait.strategy.Wait; 18 | import org.zalando.nakadiproducer.configuration.AopConfiguration; 19 | import org.zalando.nakadiproducer.configuration.TokenConfiguration; 20 | import org.zalando.nakadiproducer.event.ExampleBusinessEvent; 21 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 22 | import org.zalando.nakadiproducer.interceptor.ProfilerInterceptor; 23 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 24 | 25 | import java.io.File; 26 | import java.time.Duration; 27 | import java.util.stream.IntStream; 28 | 29 | import static org.zalando.nakadiproducer.event.ExampleBusinessEvent.EVENT_NAME; 30 | 31 | @RunWith(SpringRunner.class) 32 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) 33 | @TestPropertySource(properties = {"nakadi-producer.scheduled-transmission-enabled=false"}) 34 | @ContextConfiguration(classes = {Application.class, TokenConfiguration.class, AopConfiguration.class, ProfilerInterceptor.class}) 35 | @Slf4j 36 | public class LoadTestIT { 37 | 38 | @ClassRule 39 | public static DockerComposeContainer compose = 40 | new DockerComposeContainer(new File("src/test/resources/docker-compose.yaml")) 41 | .withExposedService("nakadi", 8080, 42 | Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(240))); 43 | 44 | @Autowired 45 | private EventLogWriter eventLogWriter; 46 | 47 | @Autowired 48 | private EventTransmitter eventTransmitter; 49 | 50 | @Autowired 51 | private RestTemplate restTemplate; 52 | 53 | @Before 54 | public void init() { 55 | createExampleEvent(); 56 | } 57 | 58 | private void createExampleEvent() { 59 | String createEvent = 60 | "{\n" + 61 | " \"name\": \"" + EVENT_NAME + "\",\n" + 62 | " \"owning_application\": \"nakadi-producer-loadtest\",\n" + 63 | " \"category\": \"undefined\",\n" + 64 | " \"partition_strategy\": \"random\",\n" + 65 | " \"schema\": {\n" + 66 | " \"type\": \"json_schema\",\n" + 67 | " \"schema\": \"{ \\\"properties\\\": { \\\"content\\\": { \\\"type\\\": \\\"string\\\" } } }\"\n" + 68 | " }" + 69 | "}"; 70 | 71 | HttpHeaders headers = new HttpHeaders(); 72 | headers.setContentType(MediaType.APPLICATION_JSON); 73 | HttpEntity entity = new HttpEntity<>(createEvent, headers); 74 | 75 | ResponseEntity response = restTemplate.exchange("http://localhost:8080/event-types", HttpMethod.POST, entity, String.class); 76 | log.debug("created event {}, response [{}]", EVENT_NAME, response); 77 | } 78 | 79 | @After 80 | public void cleanup() { 81 | deleteExampleEvent(); 82 | } 83 | 84 | private void deleteExampleEvent() { 85 | restTemplate.delete("http://localhost:8080/event-types/" + EVENT_NAME); 86 | } 87 | 88 | @Test 89 | public void testFireAndSendEvents10k() { 90 | fireBusinessEvents(10_000); 91 | sendEvents(); 92 | } 93 | 94 | @Test 95 | public void testFireAndSendEvents50k() { 96 | fireBusinessEvents(50_000); 97 | sendEvents(); 98 | } 99 | 100 | @Test 101 | public void testFireAndSendEvents100k() { 102 | fireBusinessEvents(100_000); 103 | sendEvents(); 104 | } 105 | 106 | @Test 107 | public void testFireAndSendEvents300k() { 108 | fireBusinessEvents(300_000); 109 | sendEvents(); 110 | } 111 | 112 | private void fireBusinessEvents(int amount) { 113 | log.info("=== Starting to fire " + amount + " events ============================================================"); 114 | IntStream.rangeClosed(1, amount).forEach( 115 | i -> { 116 | ExampleBusinessEvent event = new ExampleBusinessEvent("example-business-event " + i + " of " + amount); 117 | eventLogWriter.fireBusinessEvent(EVENT_NAME, event); 118 | log.debug("fired event: [{}]", event); 119 | } 120 | ); 121 | } 122 | 123 | private void sendEvents() { 124 | eventTransmitter.sendEvents(); 125 | } 126 | } -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/eventlog/impl/EventLogWriterMultipleTypesTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.jupiter.api.extension.ExtendWith; 7 | import org.mockito.ArgumentCaptor; 8 | import org.mockito.Captor; 9 | import org.mockito.Mock; 10 | import org.mockito.Mockito; 11 | import org.mockito.junit.jupiter.MockitoExtension; 12 | import org.zalando.nakadiproducer.eventlog.CompactionKeyExtractor; 13 | import org.zalando.nakadiproducer.flowid.FlowIdComponent; 14 | import org.zalando.nakadiproducer.util.Fixture; 15 | import org.zalando.nakadiproducer.util.MockPayload; 16 | 17 | import java.util.Collection; 18 | import java.util.List; 19 | 20 | import static java.util.Arrays.asList; 21 | import static java.util.stream.Collectors.toList; 22 | import static org.hamcrest.MatcherAssert.assertThat; 23 | import static org.hamcrest.Matchers.*; 24 | import static org.hamcrest.Matchers.equalTo; 25 | import static org.mockito.Mockito.verify; 26 | import static org.mockito.Mockito.when; 27 | import static org.zalando.nakadiproducer.util.Fixture.PUBLISHER_EVENT_TYPE; 28 | 29 | /** 30 | * This tests the cases where we have multiple CompactionKeyExtractors for different types. 31 | */ 32 | @ExtendWith(MockitoExtension.class) 33 | public class EventLogWriterMultipleTypesTest { 34 | 35 | @Mock 36 | private EventLogRepository eventLogRepository; 37 | 38 | @Mock 39 | private FlowIdComponent flowIdComponent; 40 | 41 | @Captor 42 | private ArgumentCaptor> eventLogsCapture; 43 | 44 | private EventLogWriterImpl eventLogWriter; 45 | 46 | private static final String TRACE_ID = "TRACE_ID"; 47 | 48 | private MockPayload eventPayload1; 49 | private MockPayload.SubClass eventPayload2; 50 | private List eventPayload3; 51 | 52 | @BeforeEach 53 | public void setUp() { 54 | Mockito.reset(eventLogRepository, flowIdComponent); 55 | 56 | eventPayload1 = Fixture.mockPayload(1, "mockedcode1", true, 57 | Fixture.mockSubClass("some info"), Fixture.mockSubList(2, "some detail")); 58 | 59 | eventPayload2 = Fixture.mockSubClass("some info"); 60 | 61 | eventPayload3 = Fixture.mockSubList(2, "some detail"); 62 | 63 | when(flowIdComponent.getXFlowIdValue()).thenReturn(TRACE_ID); 64 | } 65 | 66 | @Test 67 | public void noCompactionExtractors() { 68 | eventLogWriter = new EventLogWriterImpl(eventLogRepository, new ObjectMapper(), 69 | flowIdComponent, List.of()); 70 | 71 | eventLogWriter.fireCreateEvents(PUBLISHER_EVENT_TYPE, "", 72 | asList(eventPayload1, eventPayload2, eventPayload3)); 73 | List compactionKeys = getPersistedCompactionKeys(); 74 | assertThat(compactionKeys, contains(nullValue(), nullValue(), nullValue())); 75 | } 76 | 77 | @Test 78 | public void oneCompactionExtractor() { 79 | eventLogWriter = new EventLogWriterImpl(eventLogRepository, new ObjectMapper(), 80 | flowIdComponent, List.of(CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, MockPayload.class, m -> "Hello"))); 81 | eventLogWriter.fireCreateEvents(PUBLISHER_EVENT_TYPE, "", 82 | asList(eventPayload1, eventPayload2, eventPayload3)); 83 | List compactionKeys = getPersistedCompactionKeys(); 84 | assertThat(compactionKeys, contains(equalTo("Hello"), nullValue(), nullValue())); 85 | } 86 | 87 | @Test 88 | public void twoCompactionExtractors() { 89 | eventLogWriter = new EventLogWriterImpl(eventLogRepository, new ObjectMapper(), 90 | flowIdComponent, 91 | List.of(CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, MockPayload.class, m -> "Hello"), 92 | CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, MockPayload.SubClass.class, m -> "World"))); 93 | eventLogWriter.fireCreateEvents(PUBLISHER_EVENT_TYPE, "", 94 | asList(eventPayload1, eventPayload2, eventPayload3)); 95 | List compactionKeys = getPersistedCompactionKeys(); 96 | assertThat(compactionKeys, contains(equalTo("Hello"), equalTo("World"), nullValue())); 97 | } 98 | 99 | @Test 100 | public void threeCompactionExtractors() { 101 | eventLogWriter = new EventLogWriterImpl(eventLogRepository, new ObjectMapper(), 102 | flowIdComponent, 103 | List.of(CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, MockPayload.class, m -> "Hello"), 104 | CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, MockPayload.SubClass.class, m -> "World"), 105 | CompactionKeyExtractor.of(PUBLISHER_EVENT_TYPE, List.class, m -> "List?"))); 106 | 107 | eventLogWriter.fireCreateEvents(PUBLISHER_EVENT_TYPE, "", 108 | asList(eventPayload1, eventPayload2, eventPayload3)); 109 | List compactionKeys = getPersistedCompactionKeys(); 110 | assertThat(compactionKeys, contains(equalTo("Hello"), equalTo("World"), equalTo("List?"))); 111 | } 112 | 113 | private List getPersistedCompactionKeys() { 114 | verify(eventLogRepository).persist(eventLogsCapture.capture()); 115 | Collection eventLogs = eventLogsCapture.getValue(); 116 | List compactionKeys = eventLogs.stream().map(el -> el.getCompactionKey()).collect(toList()); 117 | return compactionKeys; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | set MAVEN_CMD_LINE_ARGS=%MAVEN_CONFIG% %* 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | 121 | set WRAPPER_JAR=""%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"" 122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 123 | 124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% 125 | if ERRORLEVEL 1 goto error 126 | goto end 127 | 128 | :error 129 | set ERROR_CODE=1 130 | 131 | :end 132 | @endlocal & set ERROR_CODE=%ERROR_CODE% 133 | 134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 138 | :skipRcPost 139 | 140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 142 | 143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 144 | 145 | exit /B %ERROR_CODE% 146 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/eventlog/impl/EventLogRepositoryImpl.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl; 2 | 3 | import java.sql.Timestamp; 4 | import java.time.Instant; 5 | import java.util.Collection; 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | import org.springframework.dao.EmptyResultDataAccessException; 11 | import org.springframework.jdbc.core.BeanPropertyRowMapper; 12 | import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; 13 | import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 14 | 15 | public class EventLogRepositoryImpl implements EventLogRepository { 16 | 17 | private NamedParameterJdbcTemplate jdbcTemplate; 18 | private int lockSize; 19 | 20 | public EventLogRepositoryImpl(NamedParameterJdbcTemplate jdbcTemplate, int lockSize) { 21 | this.jdbcTemplate = jdbcTemplate; 22 | this.lockSize = lockSize; 23 | } 24 | 25 | @Override 26 | public Collection findByLockedByAndLockedUntilGreaterThan(String lockedBy, Instant lockedUntil) { 27 | Map namedParameterMap = new HashMap<>(); 28 | namedParameterMap.put("lockedBy", lockedBy); 29 | namedParameterMap.put("lockedUntil", toSqlTimestamp(lockedUntil)); 30 | return jdbcTemplate.query( 31 | "SELECT * FROM nakadi_events.event_log where locked_by = :lockedBy and locked_until > :lockedUntil", 32 | namedParameterMap, 33 | new BeanPropertyRowMapper<>(EventLog.class) 34 | ); 35 | } 36 | 37 | @Override 38 | public void lockSomeMessages(String lockId, Instant now, Instant lockExpires) { 39 | Map namedParameterMap = new HashMap<>(); 40 | namedParameterMap.put("lockId", lockId); 41 | namedParameterMap.put("now", toSqlTimestamp(now)); 42 | namedParameterMap.put("lockExpires", toSqlTimestamp(lockExpires)); 43 | 44 | StringBuilder optionalLockSizeClause = new StringBuilder(); 45 | if (lockSize > 0) { 46 | optionalLockSizeClause.append("LIMIT :lockSize"); 47 | namedParameterMap.put("lockSize", lockSize); 48 | } 49 | 50 | jdbcTemplate.update( 51 | "UPDATE nakadi_events.event_log " 52 | + "SET locked_by = :lockId, locked_until = :lockExpires " 53 | + "WHERE id IN (SELECT id " 54 | + " FROM nakadi_events.event_log " 55 | + " WHERE locked_until IS null OR locked_until < :now " 56 | + optionalLockSizeClause 57 | + " FOR UPDATE SKIP LOCKED) ", 58 | namedParameterMap 59 | ); 60 | } 61 | 62 | @Override 63 | public void delete(EventLog eventLog) { 64 | delete(Collections.singleton(eventLog)); 65 | } 66 | 67 | @Override 68 | public void delete(Collection eventLogs) { 69 | MapSqlParameterSource[] namedParameterMaps = eventLogs.stream() 70 | .map(eventLog -> 71 | new MapSqlParameterSource().addValue("id", eventLog.getId()) 72 | ).toArray(MapSqlParameterSource[]::new); 73 | 74 | jdbcTemplate.batchUpdate( 75 | "DELETE FROM nakadi_events.event_log where id = :id", 76 | namedParameterMaps 77 | ); 78 | } 79 | 80 | @Override 81 | public void persist(EventLog eventLog) { 82 | persist(Collections.singleton(eventLog)); 83 | } 84 | 85 | @Override 86 | public void persist(Collection eventLogs) { 87 | MapSqlParameterSource[] namedParameterMaps = eventLogs.stream() 88 | .map(eventLog -> { 89 | Timestamp now = toSqlTimestamp(Instant.now()); 90 | MapSqlParameterSource namedParameterMap = new MapSqlParameterSource(); 91 | namedParameterMap.addValue("eventType", eventLog.getEventType()); 92 | namedParameterMap.addValue("eventBodyData", eventLog.getEventBodyData()); 93 | namedParameterMap.addValue("flowId", eventLog.getFlowId()); 94 | namedParameterMap.addValue("created", now); 95 | namedParameterMap.addValue("lastModified", now); 96 | namedParameterMap.addValue("lockedBy", eventLog.getLockedBy()); 97 | namedParameterMap.addValue("lockedUntil", eventLog.getLockedUntil()); 98 | namedParameterMap.addValue("compactionKey", eventLog.getCompactionKey()); 99 | return namedParameterMap; 100 | }) 101 | .toArray(MapSqlParameterSource[]::new); 102 | 103 | jdbcTemplate.batchUpdate( 104 | "INSERT INTO " + 105 | " nakadi_events.event_log " + 106 | " (event_type, event_body_data, flow_id, created, last_modified, locked_by, locked_until, compaction_key)" + 107 | "VALUES " + 108 | " (:eventType, :eventBodyData, :flowId, :created, :lastModified, :lockedBy, :lockedUntil, :compactionKey)", 109 | namedParameterMaps 110 | ); 111 | } 112 | 113 | private Timestamp toSqlTimestamp(Instant now) { 114 | if (now == null) { 115 | return null; 116 | } 117 | return Timestamp.from(now); 118 | } 119 | 120 | @Override 121 | public void deleteAll() { 122 | jdbcTemplate.update("DELETE from nakadi_events.event_log", new HashMap<>()); 123 | } 124 | 125 | @Override 126 | public EventLog findOne(Integer id) { 127 | Map namedParameterMap = new HashMap<>(); 128 | namedParameterMap.put("id", id); 129 | try { 130 | return jdbcTemplate.queryForObject( 131 | "SELECT * FROM nakadi_events.event_log where id = :id", 132 | namedParameterMap, 133 | new BeanPropertyRowMapper<>(EventLog.class) 134 | ); 135 | } catch (EmptyResultDataAccessException ignored) { 136 | return null; 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute to nakadi-producer-spring-boot-starter 2 | 3 | Thanks for contributing to our project! We really appreciate that. 4 | 5 | We are looking for all kinds of contributions, for example those: 6 | 7 | * Try out the library in your project, and tell us about your experiences. 8 | * If you see anything which could be improved, please open an issue (or comment on an existing one where it fits). 9 | * If you find something which is wrong, please open an issue, too. 10 | * If you want to implement an improvement or bugfix yourself, even better – just open a pull request. 11 | * You can also review code contributions from other people (or even from the maintainers), and/or comment on bug reports or feature requests. 12 | 13 | We are also happy about contributions to the documentation (via pull request), as well as helping to spread knowledge about our project (social media, blog posts, ...). 14 | 15 | ## Feedback 16 | 17 | ### How to report a bug 18 | 19 | **If you found a security problem which should preferably not discussed publicly, please send an email to [the maintainers](MAINTAINERS), with a copy to tech-security@zalando.de, instead of opening an issue.** 20 | 21 | Before opening a new issue for a bug report, please have a look whether there already exists a similar issue. If so, please just comment there, adding your observations. 22 | 23 | When reporting a bug, it is very helpful if you can provide enough details to reproduce it. Ideal is a complete minimal example which shows the problem. Please also mention what behavior you expect, and what you actually observe. Always include the version of the library you are using. 24 | 25 | If a bug is already fixed with the latest released version, we will likely decide not to create bugfix releases for older major/minor versions, unless those are critical security bugs. 26 | 27 | ### How to suggest a feature or enhancement 28 | 29 | We welcome all kinds of suggestions for new features or other improvements of our library. Please open an issue in the Github issue tracker. Please be as specific as you can, but if you can't, also more general suggestions are welcome. 30 | 31 | In comments to the issue we can discuss whether this is something we actually want, whether it may already exist (just not documented enough), and how it should look like in detail. 32 | 33 | Feel free to also participate in discussions to other people's enhancement requests. 34 | 35 | ### Other kinds of feedback 36 | 37 | If you just want to report your experiences, just open an issue (please include "experience report" in the title). Compared to emails to the maintainers, this is more open and visible to the public. (Although, if we find nothing actionable in there, we'll label it and then close it with a "Thank you".) 38 | 39 | ## Contributing code or documentation 40 | 41 | For bug fixes and small improvements where you already know how to implement them, you can just open a pull request with the changes. 42 | 43 | For features where you would have to invest more work, or where you need help in finding the right way of doing it, please open an issue first so we can discuss whether it goes into the direction we want to go, and so the maintainers can give you hints on where to start. (Just mention in the issue that you are willing to work on this, and whether you need some support.) 44 | 45 | If you want to contribute, but don't know what, all [issues labeled with "help wanted"](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/labels/help%20wanted) are good to start with. 46 | 47 | ### Pull requests 48 | 49 | New contributors, please fork the project, then create a branch in your fork with a suitable name, add one or more commits with your change, and open the pull request from there. Regular contributors can get write access to the main repository to be able to create feature branches there. (This makes it easier for multiple people to collaborate on one branch.) 50 | 51 | Even if you are a maintainer: never commit directly to master, always use pull-requests, so others can review your changes. 52 | 53 | Please add some text in the initial comment of the pull request describing why you are doing the change. If you have just one commit, Github will automatically fill this from the commit message, see next section. For multi-commit PRs, you'll need too summarize this manually. 54 | 55 | If you are implementing an issue, please mention its number in the PR comment, so reviewers (and people digging through the history later) can look up what this is about. (This will also create a link in the other direction, which is useful for people finding the issue first.) 56 | 57 | ### Code Style 58 | We do not follow a particular standard here. Please have a look around in the files you are changing and respect and adapt to the code style you find there. 59 | 60 | ### Commit messages 61 | 62 | Please try to explain why you did your changes in the commit messages. If you are implementing an issue, include its number in the message. 63 | 64 | ### Tests 65 | 66 | If you are fixing a bug, please try to also add a test which would fail without the fix, thereby making sure it doesn't happen again. If you are adding a new feature, please also add tests for it. 67 | 68 | The changed code from your pull request will be automatically run on our continuous integration system (Travis CI), it will send its feedback back, and a non-failing build + tests is a requirement for merging. (You can click on the link to see what did go wrong.) 69 | 70 | ### Code review 71 | 72 | For a pull request to be merged, it needs at least two comments with just a ":+1:" in them (you can type `:+1:`) from members of the Zalando Github organization (i.e. software engineers at Zalando), then a maintainer can merge it. We also appreciate other interested persons to review our and other's code (but for compliance reasons, their :+1: don't count). 73 | 74 | Just as for issues, the maintainers try to respond to every pull request in 72 hours (excluding weekends). 75 | 76 | ## Conduct 77 | 78 | In the interest of fostering an open and welcoming environment, we follow and enforce our [Code of Conduct](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/blob/master/CODE_OF_CONDUCT.md). 79 | -------------------------------------------------------------------------------- /nakadi-producer/src/test/java/org/zalando/nakadiproducer/transmission/impl/EventBatcherTest.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.junit.jupiter.api.Test; 6 | import org.zalando.nakadiproducer.eventlog.impl.EventLog; 7 | import org.zalando.nakadiproducer.transmission.impl.EventBatcher.BatchItem; 8 | 9 | import java.util.List; 10 | import java.util.function.Consumer; 11 | 12 | import static java.time.Instant.now; 13 | import static java.util.Arrays.asList; 14 | import static java.util.Collections.singletonList; 15 | import static org.mockito.ArgumentMatchers.any; 16 | import static org.mockito.ArgumentMatchers.eq; 17 | import static org.mockito.Mockito.mock; 18 | import static org.mockito.Mockito.never; 19 | import static org.mockito.Mockito.verify; 20 | import static org.mockito.Mockito.when; 21 | 22 | public class EventBatcherTest { 23 | private final ObjectMapper objectMapper = mock(ObjectMapper.class); 24 | private final Consumer> publisher = mock(Consumer.class); 25 | private final EventBatcher eventBatcher = new EventBatcher(objectMapper, publisher); 26 | 27 | @Test 28 | public void shouldNotPublishEmptyBatches() { 29 | eventBatcher.finish(); 30 | 31 | verify(publisher, never()).accept(any()); 32 | } 33 | 34 | @Test 35 | public void shouldPublishNonFilledBatchOnFinish() throws JsonProcessingException { 36 | EventLog eventLogEntry = eventLogEntry(1, "type"); 37 | NakadiEvent nakadiEvent = nakadiEvent("1"); 38 | 39 | when(objectMapper.writeValueAsBytes(any())).thenReturn(new byte[500]); 40 | 41 | eventBatcher.pushEvent(eventLogEntry, nakadiEvent); 42 | verify(publisher, never()).accept(any()); 43 | 44 | eventBatcher.finish(); 45 | verify(publisher).accept(eq(singletonList(new BatchItem(eventLogEntry, nakadiEvent)))); 46 | } 47 | 48 | @Test 49 | public void shouldPublishNonFilledBatchOnEventTypeChange() throws JsonProcessingException { 50 | EventLog eventLogEntry1 = eventLogEntry(1, "type1"); 51 | EventLog eventLogEntry2 = eventLogEntry(2, "type2"); 52 | NakadiEvent nakadiEvent1 = nakadiEvent("1"); 53 | NakadiEvent nakadiEvent2 = nakadiEvent("2"); 54 | 55 | when(objectMapper.writeValueAsBytes(any())).thenReturn(new byte[500]); 56 | 57 | eventBatcher.pushEvent(eventLogEntry1, nakadiEvent1); 58 | eventBatcher.pushEvent(eventLogEntry2, nakadiEvent2); 59 | verify(publisher).accept(eq(singletonList(new BatchItem(eventLogEntry1, nakadiEvent1)))); 60 | } 61 | 62 | @Test 63 | public void shouldPublishFilledBatchOnSubmissionOfNewEvent() throws JsonProcessingException { 64 | EventLog eventLogEntry1 = eventLogEntry(1, "type1"); 65 | EventLog eventLogEntry2 = eventLogEntry(2, "type1"); 66 | EventLog eventLogEntry3 = eventLogEntry(3, "type1"); 67 | NakadiEvent nakadiEvent1 = nakadiEvent("1"); 68 | NakadiEvent nakadiEvent2 = nakadiEvent("2"); 69 | NakadiEvent nakadiEvent3 = nakadiEvent("3"); 70 | 71 | when(objectMapper.writeValueAsBytes(any())).thenReturn(new byte[15000000]); 72 | 73 | // 15 MB batch size 74 | eventBatcher.pushEvent(eventLogEntry1, nakadiEvent1); 75 | // 30 MB batch size 76 | eventBatcher.pushEvent(eventLogEntry2, nakadiEvent2); 77 | // would be 45MB batch size, which is more than 80% of 50MB, therefore triggers submission of the previous two 78 | eventBatcher.pushEvent(eventLogEntry3, nakadiEvent3); 79 | 80 | verify(publisher) 81 | .accept(eq(asList( 82 | new BatchItem(eventLogEntry1, nakadiEvent1), 83 | new BatchItem(eventLogEntry2, nakadiEvent2) 84 | ))); 85 | } 86 | 87 | @Test 88 | public void shouldTryPublishEventsIndividuallyWhenTheyExceedBatchThresholdThe() throws JsonProcessingException { 89 | EventLog eventLogEntry1 = eventLogEntry(1, "type1"); 90 | EventLog eventLogEntry2 = eventLogEntry(2, "type1"); 91 | NakadiEvent nakadiEvent1 = nakadiEvent("1"); 92 | NakadiEvent nakadiEvent2 = nakadiEvent("2"); 93 | 94 | when(objectMapper.writeValueAsBytes(any())) 95 | .thenReturn(new byte[45000000]) 96 | .thenReturn(new byte[450]); 97 | 98 | // 45 MB batch size => will form a batch of its own 99 | eventBatcher.pushEvent(eventLogEntry1, nakadiEvent1); 100 | // ... and be submitted with the next event added 101 | eventBatcher.pushEvent(eventLogEntry2, nakadiEvent2); 102 | 103 | verify(publisher).accept(eq(singletonList(new BatchItem(eventLogEntry1, nakadiEvent1)))); 104 | } 105 | 106 | @Test 107 | public void willGracefullySkipNonSerializableEvents() throws JsonProcessingException { 108 | EventLog eventLogEntry1 = eventLogEntry(1, "type1"); 109 | EventLog eventLogEntry2 = eventLogEntry(2, "type1"); 110 | NakadiEvent nakadiEvent1 = nakadiEvent("1"); 111 | NakadiEvent nakadiEvent2 = nakadiEvent("2"); 112 | 113 | when(objectMapper.writeValueAsBytes(any())) 114 | .thenThrow(new IllegalStateException()) 115 | .thenReturn(new byte[450]); 116 | 117 | // non serializable 118 | eventBatcher.pushEvent(eventLogEntry1, nakadiEvent1); 119 | // serializable 120 | eventBatcher.pushEvent(eventLogEntry2, nakadiEvent2); 121 | // and flush it 122 | eventBatcher.finish(); 123 | 124 | verify(publisher).accept(eq(singletonList(new BatchItem(eventLogEntry2, nakadiEvent2)))); 125 | } 126 | 127 | private EventLog eventLogEntry(int id, String type) { 128 | return new EventLog(id, type, "body", "flow", now(), now(), "me", now(), null); 129 | } 130 | 131 | private NakadiEvent nakadiEvent(String eid) { 132 | NakadiMetadata metadata = new NakadiMetadata(); 133 | metadata.setEid(eid); 134 | NakadiEvent nakadiEvent = new NakadiEvent(); 135 | nakadiEvent.setMetadata(metadata); 136 | return nakadiEvent; 137 | } 138 | } -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/eventlog/impl/batcher/QueryStatementBatcherIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.eventlog.impl.batcher; 2 | 3 | import org.junit.jupiter.api.AfterEach; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Disabled; 6 | import org.junit.jupiter.api.Test; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.jdbc.core.RowMapper; 9 | import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; 10 | import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 11 | import org.zalando.nakadiproducer.BaseMockedExternalCommunicationIT; 12 | 13 | import java.time.Duration; 14 | import java.time.Instant; 15 | import java.util.Arrays; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.stream.IntStream; 19 | 20 | import static java.util.stream.Collectors.toList; 21 | import static org.hamcrest.MatcherAssert.assertThat; 22 | import static org.hamcrest.Matchers.hasSize; 23 | import static org.hamcrest.Matchers.is; 24 | 25 | public class QueryStatementBatcherIT extends BaseMockedExternalCommunicationIT { 26 | 27 | private static final RowMapper ID_ROW_MAPPER = (row, n) -> row.getInt("id"); 28 | @Autowired 29 | private NamedParameterJdbcTemplate jdbcTemplate; 30 | 31 | @BeforeEach 32 | public void setUpTable() { 33 | jdbcTemplate.update("CREATE TABLE x (id SERIAL, a INT, b INT)", Map.of()); 34 | } 35 | @AfterEach 36 | public void dropTable() { 37 | jdbcTemplate.update("DROP TABLE x;", Map.of()); 38 | } 39 | 40 | @Test 41 | public void testStreamEvents() { 42 | QueryStatementBatcher batcher = createInsertPairsReturningIdBatcher(); 43 | MapSqlParameterSource commonArguments = new MapSqlParameterSource(); 44 | int expectedCount = 31; 45 | List repeatedInputs = IntStream.range(0, expectedCount) 46 | .mapToObj(i -> new MapSqlParameterSource() 47 | .addValue("a#", i) 48 | .addValue("b#", 5 * i)) 49 | .collect(toList()); 50 | 51 | List resultList = batcher.queryForStream( 52 | jdbcTemplate, repeatedInputs.stream()) 53 | .collect(toList()); 54 | assertThat(resultList, hasSize(expectedCount)); 55 | 56 | List secondResultList = batcher.queryForStream( 57 | jdbcTemplate, commonArguments, repeatedInputs.stream()) 58 | .collect(toList()); 59 | 60 | assertThat(secondResultList, hasSize(expectedCount)); 61 | assertThat(secondResultList.get(0), is(expectedCount+1)); 62 | } 63 | 64 | private static QueryStatementBatcher createInsertPairsReturningIdBatcher() { 65 | return new QueryStatementBatcher<>( 66 | "INSERT INTO x (a, b) VALUES ", "(:a#, :b#)", " RETURNING id", 67 | ID_ROW_MAPPER, 68 | 51, 13, 4, 1); 69 | } 70 | 71 | @Test 72 | @Disabled("Running benchmarks takes too long.") 73 | public void benchmarkWithBatcher() { 74 | int totalCount = 5000; 75 | List inputs = prepareInputs(totalCount); 76 | Instant before = Instant.now(); 77 | QueryStatementBatcher batcher = createInsertPairsReturningIdBatcher(); 78 | List results = batcher.queryForStream(jdbcTemplate, inputs.stream()).collect(toList()); 79 | Instant after = Instant.now(); 80 | System.err.format("Inserting %s items took %s.\n", totalCount, Duration.between(before, after)); 81 | System.out.println(results); 82 | } 83 | 84 | private static List prepareInputs(int totalCount) { 85 | return IntStream.range(0, totalCount) 86 | .mapToObj(i -> new MapSqlParameterSource() 87 | .addValue("a#", 3 * i) 88 | .addValue("b#", 5 * i)) 89 | .collect(toList()); 90 | } 91 | 92 | @Test 93 | @Disabled("Running benchmarks takes too long.") 94 | public void benchmarkWithoutBatcherSerial() { 95 | int totalCount = 5000; 96 | List inputs = prepareInputs(totalCount); 97 | Instant before = Instant.now(); 98 | List results = inputs.stream() 99 | .map(source -> jdbcTemplate.queryForObject( 100 | "INSERT INTO x (a, b) VALUES (:a#, :b#) RETURNING id", 101 | source, ID_ROW_MAPPER)) 102 | .collect(toList()); 103 | Instant after = Instant.now(); 104 | System.err.format("Inserting %s items took %s.\n", totalCount, Duration.between(before, after)); 105 | System.out.println(results); 106 | } 107 | 108 | @Test 109 | @Disabled("Running benchmarks takes too long.") 110 | public void benchmarkWithoutBatcherParallel() { 111 | int totalCount = 5000; 112 | List inputs = prepareInputs(totalCount); 113 | Instant before = Instant.now(); 114 | List results = inputs.parallelStream() 115 | .map(source -> jdbcTemplate.queryForObject( 116 | "INSERT INTO x (a, b) VALUES (:a#, :b#) RETURNING id", 117 | source, ID_ROW_MAPPER)) 118 | .collect(toList()); 119 | Instant after = Instant.now(); 120 | System.err.format("Inserting %s items took %s.\n", totalCount, Duration.between(before, after)); 121 | System.out.println(results); 122 | } 123 | 124 | @Test 125 | @Disabled("Running benchmarks takes too long.") 126 | public void benchmarkBatchWithoutReturn() { 127 | int totalCount = 5000; 128 | List inputs = prepareInputs(totalCount); 129 | MapSqlParameterSource[] inputArray = inputs.toArray(new MapSqlParameterSource[0]); 130 | Instant before = Instant.now(); 131 | int[] results = jdbcTemplate.batchUpdate( 132 | "INSERT INTO x (a, b) VALUES (:a#, :b#)", 133 | inputArray); 134 | Instant after = Instant.now(); 135 | System.err.format("Inserting %s items took %s.\n", totalCount, Duration.between(before, after)); 136 | System.out.println(Arrays.toString(results)); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/EndToEndTestIT.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import static com.jayway.jsonpath.Criteria.where; 4 | import static com.jayway.jsonpath.JsonPath.read; 5 | import static org.hamcrest.MatcherAssert.assertThat; 6 | import static org.hamcrest.Matchers.*; 7 | 8 | import java.io.IOException; 9 | import java.util.List; 10 | 11 | import org.junit.jupiter.api.AfterEach; 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.context.ApplicationContext; 16 | import org.springframework.context.annotation.Bean; 17 | import org.springframework.test.context.ContextConfiguration; 18 | import org.zalando.nakadiproducer.eventlog.CompactionKeyExtractor; 19 | import org.zalando.nakadiproducer.eventlog.EventLogWriter; 20 | import org.zalando.nakadiproducer.transmission.MockNakadiPublishingClient; 21 | import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; 22 | import org.zalando.nakadiproducer.util.Fixture; 23 | import org.zalando.nakadiproducer.util.MockPayload; 24 | 25 | @ContextConfiguration(classes = EndToEndTestIT.Config.class) 26 | public class EndToEndTestIT extends BaseMockedExternalCommunicationIT { 27 | private static final String MY_DATA_CHANGE_EVENT_TYPE = "myDataChangeEventType"; 28 | private static final String SECOND_DATA_CHANGE_EVENT_TYPE = "secondDataChangeEventType"; 29 | private static final String MY_BUSINESS_EVENT_TYPE = "myBusinessEventType"; 30 | public static final String PUBLISHER_DATA_TYPE = "nakadi:some-publisher"; 31 | private static final String CODE = "code123"; 32 | public static final String COMPACTION_KEY = "Hello World"; 33 | 34 | @Autowired 35 | private EventLogWriter eventLogWriter; 36 | 37 | @Autowired 38 | private EventTransmitter eventTransmitter; 39 | 40 | @Autowired 41 | private MockNakadiPublishingClient nakadiClient; 42 | 43 | @Autowired 44 | ApplicationContext context; 45 | 46 | @BeforeEach 47 | @AfterEach 48 | public void clearNakadiEvents() { 49 | eventTransmitter.sendEvents(); 50 | nakadiClient.clearSentEvents(); 51 | } 52 | 53 | @Test 54 | public void noStupsTokenBeanIsSetupWithMockPublishingClient() { 55 | assertThat(context.getBeanProvider(StupsTokenComponent.class).getIfAvailable(), is(nullValue())); 56 | } 57 | 58 | @Test 59 | public void dataEventsShouldBeSubmittedToNakadi() throws IOException { 60 | MockPayload payload = Fixture.mockPayload(1, CODE); 61 | eventLogWriter.fireCreateEvent(MY_DATA_CHANGE_EVENT_TYPE, PUBLISHER_DATA_TYPE, payload); 62 | 63 | eventTransmitter.sendEvents(); 64 | List value = nakadiClient.getSentEvents(MY_DATA_CHANGE_EVENT_TYPE); 65 | 66 | assertThat(value.size(), is(1)); 67 | assertThat(read(value.get(0), "$.data_op"), is("C")); 68 | assertThat(read(value.get(0), "$.data_type"), is(PUBLISHER_DATA_TYPE)); 69 | assertThat(read(value.get(0), "$.data.code"), is(CODE)); 70 | } 71 | 72 | @Test 73 | public void compactionKeyIsPreserved() throws IOException { 74 | MockPayload payload = Fixture.mockPayload(1, CODE); 75 | eventLogWriter.fireDeleteEvent(SECOND_DATA_CHANGE_EVENT_TYPE, PUBLISHER_DATA_TYPE, payload); 76 | eventLogWriter.fireBusinessEvent(MY_BUSINESS_EVENT_TYPE, payload); 77 | 78 | eventTransmitter.sendEvents(); 79 | 80 | List dataEvents = nakadiClient.getSentEvents(SECOND_DATA_CHANGE_EVENT_TYPE); 81 | assertThat(dataEvents.size(), is(1)); 82 | assertThat(read(dataEvents.get(0), "$.metadata.partition_compaction_key"), is(COMPACTION_KEY)); 83 | 84 | List businessEvents = nakadiClient.getSentEvents(MY_BUSINESS_EVENT_TYPE); 85 | assertThat(businessEvents.size(), is(1)); 86 | assertThat(read(businessEvents.get(0), "$.metadata.partition_compaction_key"), is(CODE)); 87 | } 88 | 89 | @Test 90 | public void compactionKeyIsNotInvented() throws IOException { 91 | MockPayload payload = Fixture.mockPayload(1, CODE); 92 | eventLogWriter.fireDeleteEvent(MY_DATA_CHANGE_EVENT_TYPE, PUBLISHER_DATA_TYPE, payload); 93 | 94 | eventTransmitter.sendEvents(); 95 | List value = nakadiClient.getSentEvents(MY_DATA_CHANGE_EVENT_TYPE); 96 | 97 | assertThat(value.size(), is(1)); 98 | assertThat(read(value.get(0), "$.metadata[?]", where("partition_compaction_key").exists(true)), 99 | is(empty())); 100 | } 101 | 102 | @Test 103 | public void businessEventsShouldBeSubmittedToNakadi() throws IOException { 104 | MockPayload payload = Fixture.mockPayload(1, CODE); 105 | eventLogWriter.fireBusinessEvent(MY_BUSINESS_EVENT_TYPE, payload); 106 | 107 | eventTransmitter.sendEvents(); 108 | List value = nakadiClient.getSentEvents(MY_BUSINESS_EVENT_TYPE); 109 | 110 | assertThat(value.size(), is(1)); 111 | assertThat(read(value.get(0), "$.id"), is(payload.getId())); 112 | assertThat(read(value.get(0), "$.code"), is(payload.getCode())); 113 | assertThat(read(value.get(0), "$.items.length()"), is(3)); 114 | assertThat(read(value.get(0), "$.items[0].detail"), is(payload.getItems().get(0).getDetail())); 115 | assertThat(read(value.get(0), "$.items[1].detail"), is(payload.getItems().get(1).getDetail())); 116 | assertThat(read(value.get(0), "$.items[2].detail"), is(payload.getItems().get(2).getDetail())); 117 | assertThat(read(value.get(0), "$.more.info"), is(payload.getMore().getInfo())); 118 | assertThat(read(value.get(0), "$[?]", where("data_op").exists(true)), is(empty())); 119 | assertThat(read(value.get(0), "$[?]", where("data_type").exists(true)), is(empty())); 120 | assertThat(read(value.get(0), "$[?]", where("data").exists(true)), is(empty())); 121 | } 122 | 123 | public static class Config { 124 | @Bean 125 | public CompactionKeyExtractor compactionKeyExtractorForSecondDataEventType() { 126 | return CompactionKeyExtractor.of(SECOND_DATA_CHANGE_EVENT_TYPE, MockPayload.class, m -> COMPACTION_KEY); 127 | } 128 | 129 | @Bean 130 | public CompactionKeyExtractor keyExtractorForBusinessEventType() { 131 | return CompactionKeyExtractor.of(MY_BUSINESS_EVENT_TYPE, MockPayload.class, MockPayload::getCode); 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /nakadi-producer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | 3.2 8 | 9 | 10 | 11 | org.zalando 12 | nakadi-producer-reactor 13 | 21.0.0 14 | 15 | 16 | nakadi-producer 17 | ${project.parent.version} 18 | Nakadi Event Producer 19 | Reliable transactional Nakadi event producer 20 | 21 | 22 | 11 23 | 11 24 | 11 25 | 26 | 27 | 28 | 29 | javax.transaction 30 | javax.transaction-api 31 | 32 | 33 | com.fasterxml.jackson.core 34 | jackson-databind 35 | 36 | 37 | com.fasterxml.jackson.datatype 38 | jackson-datatype-jsr310 39 | 40 | 41 | org.slf4j 42 | slf4j-api 43 | 44 | 45 | org.zalando 46 | fahrschein 47 | 0.24.0 48 | 49 | 50 | org.projectlombok 51 | lombok 52 | provided 53 | 54 | 58 | 59 | javax.interceptor 60 | javax.interceptor-api 61 | 1.2.2 62 | provided 63 | true 64 | 65 | 66 | org.hamcrest 67 | hamcrest-all 68 | 1.3 69 | test 70 | 71 | 72 | org.mockito 73 | mockito-core 74 | test 75 | 76 | 77 | com.jayway.jsonpath 78 | json-path 79 | test 80 | 81 | 82 | org.junit.jupiter 83 | junit-jupiter 84 | test 85 | 86 | 87 | org.springframework 88 | spring-core 89 | 5.3.27 90 | 91 | 92 | org.mockito 93 | mockito-junit-jupiter 94 | 3.9.0 95 | test 96 | 97 | 98 | 99 | 100 | ${project.artifactId} 101 | 102 | 103 | 104 | org.apache.maven.plugins 105 | maven-failsafe-plugin 106 | 107 | 108 | 109 | integration-test 110 | verify 111 | 112 | 113 | 114 | 115 | 116 | org.apache.maven.plugins 117 | maven-source-plugin 118 | 119 | 120 | attach-sources 121 | 122 | jar-no-fork 123 | 124 | 125 | 126 | 127 | 128 | org.apache.maven.plugins 129 | maven-javadoc-plugin 130 | 131 | -Xdoclint:none 132 | 133 | 134 | 135 | attach-javadocs 136 | 137 | jar 138 | 139 | 140 | 141 | 142 | 143 | org.apache.maven.plugins 144 | maven-gpg-plugin 145 | 1.5 146 | 147 | 148 | sign-artifacts 149 | verify 150 | 151 | sign 152 | 153 | 154 | 155 | 156 | 157 | org.sonatype.plugins 158 | nexus-staging-maven-plugin 159 | 1.6.7 160 | true 161 | 162 | ossrh 163 | https://oss.sonatype.org/ 164 | true 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | MIT 173 | https://opensource.org/licenses/MIT 174 | repo 175 | 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /nakadi-producer/src/main/java/org/zalando/nakadiproducer/transmission/impl/EventTransmissionService.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer.transmission.impl; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.zalando.fahrschein.EventPublishingException; 7 | import org.zalando.fahrschein.domain.BatchItemResponse; 8 | import org.zalando.nakadiproducer.eventlog.impl.EventLog; 9 | import org.zalando.nakadiproducer.eventlog.impl.EventLogRepository; 10 | import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; 11 | import org.zalando.nakadiproducer.transmission.impl.EventBatcher.BatchItem; 12 | 13 | import javax.transaction.Transactional; 14 | 15 | import java.io.IOException; 16 | import java.time.Clock; 17 | import java.time.Instant; 18 | import java.util.Arrays; 19 | import java.util.Collection; 20 | import java.util.LinkedHashMap; 21 | import java.util.List; 22 | import java.util.UUID; 23 | import java.util.stream.Collectors; 24 | import java.util.stream.Stream; 25 | 26 | import static java.time.temporal.ChronoUnit.SECONDS; 27 | 28 | @Slf4j 29 | public class EventTransmissionService { 30 | 31 | private final EventLogRepository eventLogRepository; 32 | private final NakadiPublishingClient nakadiPublishingClient; 33 | private final ObjectMapper objectMapper; 34 | private final int lockDuration; 35 | private final int lockDurationBuffer; 36 | 37 | private Clock clock = Clock.systemDefaultZone(); 38 | 39 | public EventTransmissionService(EventLogRepository eventLogRepository, NakadiPublishingClient nakadiPublishingClient, ObjectMapper objectMapper, 40 | int lockDuration, int lockDurationBuffer) { 41 | this.eventLogRepository = eventLogRepository; 42 | this.nakadiPublishingClient = nakadiPublishingClient; 43 | this.objectMapper = objectMapper; 44 | this.lockDuration = lockDuration; 45 | this.lockDurationBuffer = lockDurationBuffer; 46 | } 47 | 48 | @Transactional 49 | public Collection lockSomeEvents() { 50 | String lockId = UUID.randomUUID().toString(); 51 | log.debug("Locking events for replication with lockId {} for {} seconds", lockId, lockDuration); 52 | eventLogRepository.lockSomeMessages(lockId, now(), now().plus(lockDuration, SECONDS)); 53 | return eventLogRepository.findByLockedByAndLockedUntilGreaterThan(lockId, now()); 54 | } 55 | 56 | @Transactional 57 | public void sendEvents(Collection events) { 58 | EventBatcher batcher = new EventBatcher(objectMapper, this::publishBatch); 59 | 60 | for (EventLog event : events) { 61 | if (lockNearlyExpired(event)) { 62 | // to avoid that two instances process this event, we skip it 63 | continue; 64 | } 65 | 66 | NakadiEvent nakadiEvent; 67 | 68 | try { 69 | nakadiEvent = mapToNakadiEvent(event); 70 | } catch (Exception e) { 71 | log.error("Could not serialize event {} of type {}, skipping it.", event.getId(), event.getEventType(), e); 72 | continue; 73 | } 74 | 75 | batcher.pushEvent(event, nakadiEvent); 76 | } 77 | 78 | batcher.finish(); 79 | } 80 | 81 | /** 82 | * Publishes a list of events. 83 | * All of the events in this list need to be destined for the same event type. 84 | */ 85 | private void publishBatch(List batch) { 86 | try { 87 | this.tryToPublishBatch(batch); 88 | } catch (Exception e) { 89 | log.error("Could not send {} events of type {}, skipping them.", batch.size(), batch.get(0).getEventLogEntry().getEventType(), e); 90 | } 91 | } 92 | 93 | /** 94 | * Tries to publish a set of events (all of which need to belong to the same event type). 95 | * The successful ones will be deleted from the database. 96 | */ 97 | private void tryToPublishBatch(List batch) throws Exception { 98 | Stream successfulEvents; 99 | String eventType = batch.get(0).getEventLogEntry().getEventType(); 100 | try { 101 | nakadiPublishingClient.publish( 102 | eventType, 103 | batch.stream() 104 | .map(BatchItem::getNakadiEvent) 105 | .collect(Collectors.toList()) 106 | ); 107 | successfulEvents = batch.stream().map(BatchItem::getEventLogEntry); 108 | log.info("Sent {} events of type {}.", batch.size(), eventType); 109 | } catch (EventPublishingException e) { 110 | log.error("{} out of {} events of type {} failed to be sent. Exception ", 111 | e.getResponses().length, batch.size(), eventType, e); 112 | List failedEids = collectEids(e); 113 | successfulEvents = 114 | batch.stream() 115 | .map(BatchItem::getEventLogEntry) 116 | .filter(rawEvent -> !failedEids.contains(convertToUUID(rawEvent.getId()))); 117 | } 118 | 119 | eventLogRepository.delete(successfulEvents.collect(Collectors.toList())); 120 | } 121 | 122 | private List collectEids(EventPublishingException e) { 123 | return Arrays.stream(e.getResponses()).map(BatchItemResponse::getEid).collect(Collectors.toList()); 124 | } 125 | 126 | private boolean lockNearlyExpired(EventLog eventLog) { 127 | // since clocks never work exactly synchronous and sending the event also takes some time, we include a safety 128 | // buffer here. This is still not 100% precise, but since we require events to be consumed idempotent, sending 129 | // one event twice won't hurt much. 130 | return now().isAfter(eventLog.getLockedUntil().minus(lockDurationBuffer, SECONDS)); 131 | } 132 | 133 | private NakadiEvent mapToNakadiEvent(final EventLog event) throws IOException { 134 | final NakadiEvent nakadiEvent = new NakadiEvent(); 135 | 136 | final NakadiMetadata metadata = new NakadiMetadata(); 137 | metadata.setEid(convertToUUID(event.getId())); 138 | metadata.setOccuredAt(event.getCreated()); 139 | metadata.setFlowId(event.getFlowId()); 140 | metadata.setPartitionCompactionKey(event.getCompactionKey()); 141 | nakadiEvent.setMetadata(metadata); 142 | 143 | LinkedHashMap payloadDTO = objectMapper.readValue(event.getEventBodyData(), new TypeReference>() { }); 144 | 145 | nakadiEvent.setData(payloadDTO); 146 | 147 | return nakadiEvent; 148 | } 149 | 150 | private Instant now() { 151 | return clock.instant(); 152 | } 153 | 154 | public void overrideClock(Clock clock) { 155 | this.clock = clock; 156 | } 157 | 158 | /** 159 | * Converts a number in UUID format. 160 | * 161 | *

For instance 213 will be converted to "00000000-0000-0000-0000-0000000000d5"

162 | */ 163 | private String convertToUUID(final int number) { 164 | return new UUID(0, number).toString(); 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | 3.2 8 | 9 | 10 | 11 | org.zalando 12 | nakadi-producer-reactor 13 | 21.0.0 14 | 15 | 16 | nakadi-producer-spring-boot-starter 17 | ${project.parent.version} 18 | Nakadi Event Producer: Spring Boot Starter 19 | Spring Boot Auto Configuration for Nakadi event producer 20 | 21 | 22 | 11 23 | 11 24 | 11 25 | 26 | 27 | 28 | 29 | org.zalando 30 | nakadi-producer 31 | ${project.version} 32 | 33 | 34 | org.springframework 35 | spring-jdbc 36 | 37 | 38 | org.flywaydb 39 | flyway-core 40 | 41 | 42 | org.projectlombok 43 | lombok 44 | provided 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-actuator 49 | true 50 | 51 | 52 | org.zalando 53 | tracer-core 54 | 0.17.2 55 | true 56 | 57 | 58 | org.zalando.stups 59 | tokens 60 | 0.14.0 61 | true 62 | 63 | 64 | org.springframework.boot 65 | spring-boot-starter-test 66 | test 67 | 68 | 69 | org.postgresql 70 | postgresql 71 | 42.4.3 72 | test 73 | 74 | 75 | io.zonky.test 76 | embedded-postgres 77 | 2.0.4 78 | test 79 | 80 | 81 | commons-logging 82 | commons-logging 83 | 1.2 84 | test 85 | 86 | 87 | com.jayway.restassured 88 | rest-assured 89 | 2.4.0 90 | test 91 | 92 | 93 | org.springframework.boot 94 | spring-boot-starter-web 95 | test 96 | 97 | 98 | org.springframework.boot 99 | spring-boot-autoconfigure 100 | 101 | 102 | org.springframework.boot 103 | spring-boot-actuator-autoconfigure 104 | 105 | 106 | jakarta.annotation 107 | jakarta.annotation-api 108 | 109 | 110 | 111 | 112 | 113 | ${project.artifactId} 114 | 115 | 116 | 117 | org.apache.maven.plugins 118 | maven-failsafe-plugin 119 | 120 | 121 | 122 | integration-test 123 | verify 124 | 125 | 126 | 127 | 128 | 129 | org.apache.maven.plugins 130 | maven-source-plugin 131 | 2.2.1 132 | 133 | 134 | attach-sources 135 | 136 | jar-no-fork 137 | 138 | 139 | 140 | 141 | 142 | org.apache.maven.plugins 143 | maven-javadoc-plugin 144 | 2.9.1 145 | 146 | -Xdoclint:none 147 | 148 | 149 | 150 | attach-javadocs 151 | 152 | jar 153 | 154 | 155 | 156 | 157 | 158 | org.apache.maven.plugins 159 | maven-gpg-plugin 160 | 1.5 161 | 162 | 163 | sign-artifacts 164 | verify 165 | 166 | sign 167 | 168 | 169 | 170 | 171 | 172 | org.sonatype.plugins 173 | nexus-staging-maven-plugin 174 | 1.6.7 175 | true 176 | 177 | ossrh 178 | https://oss.sonatype.org/ 179 | true 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | MIT 188 | https://opensource.org/licenses/MIT 189 | repo 190 | 191 | 192 | 193 | 194 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | if [ -d "$wdir"/.mvn ] ; then 204 | basedir=$wdir 205 | break 206 | fi 207 | wdir=$(cd "$wdir/.."; pwd) 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} $MAVEN_CMD_LINE_ARGS 234 | 235 | -------------------------------------------------------------------------------- /nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java: -------------------------------------------------------------------------------- 1 | package org.zalando.nakadiproducer; 2 | 3 | import javax.annotation.PostConstruct; 4 | import javax.sql.DataSource; 5 | 6 | import org.flywaydb.core.Flyway; 7 | import org.flywaydb.core.api.callback.BaseCallback; 8 | import org.flywaydb.core.api.callback.Context; 9 | import org.flywaydb.core.api.callback.Event; 10 | import org.flywaydb.core.api.configuration.FluentConfiguration; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.boot.autoconfigure.flyway.FlywayDataSource; 13 | import org.springframework.boot.autoconfigure.flyway.FlywayProperties; 14 | import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; 15 | import java.util.List; 16 | import java.util.Optional; 17 | import java.util.Set; 18 | import java.util.stream.Collectors; 19 | import java.util.stream.Stream; 20 | 21 | import static org.flywaydb.core.api.callback.Event.AFTER_BASELINE; 22 | import static org.flywaydb.core.api.callback.Event.AFTER_CLEAN; 23 | import static org.flywaydb.core.api.callback.Event.AFTER_EACH_MIGRATE; 24 | import static org.flywaydb.core.api.callback.Event.AFTER_EACH_UNDO; 25 | import static org.flywaydb.core.api.callback.Event.AFTER_INFO; 26 | import static org.flywaydb.core.api.callback.Event.AFTER_MIGRATE; 27 | import static org.flywaydb.core.api.callback.Event.AFTER_REPAIR; 28 | import static org.flywaydb.core.api.callback.Event.AFTER_UNDO; 29 | import static org.flywaydb.core.api.callback.Event.AFTER_VALIDATE; 30 | import static org.flywaydb.core.api.callback.Event.BEFORE_BASELINE; 31 | import static org.flywaydb.core.api.callback.Event.BEFORE_CLEAN; 32 | import static org.flywaydb.core.api.callback.Event.BEFORE_EACH_MIGRATE; 33 | import static org.flywaydb.core.api.callback.Event.BEFORE_EACH_UNDO; 34 | import static org.flywaydb.core.api.callback.Event.BEFORE_INFO; 35 | import static org.flywaydb.core.api.callback.Event.BEFORE_MIGRATE; 36 | import static org.flywaydb.core.api.callback.Event.BEFORE_REPAIR; 37 | import static org.flywaydb.core.api.callback.Event.BEFORE_UNDO; 38 | import static org.flywaydb.core.api.callback.Event.BEFORE_VALIDATE; 39 | 40 | public class FlywayMigrator { 41 | @Autowired(required = false) 42 | @NakadiProducerFlywayDataSource 43 | private DataSource nakadiProducerFlywayDataSource; 44 | 45 | @Autowired(required = false) 46 | @FlywayDataSource 47 | private DataSource flywayDataSource; 48 | 49 | @Autowired 50 | private DataSource dataSource; 51 | 52 | @Autowired(required = false) 53 | private List callbacks; 54 | 55 | @Autowired 56 | private FlywayProperties flywayProperties; 57 | 58 | @Autowired 59 | private DataSourceProperties dataSourceProperties; 60 | 61 | @PostConstruct 62 | public void migrateFlyway() { 63 | final FluentConfiguration config = Flyway.configure(); 64 | 65 | if (this.nakadiProducerFlywayDataSource != null) { 66 | config.dataSource(nakadiProducerFlywayDataSource); 67 | } else if (this.flywayProperties != null && (flywayProperties.getUser() != null || flywayProperties.getUrl() != null)) { 68 | config.dataSource( 69 | Optional.ofNullable(this.flywayProperties.getUrl()).orElse(dataSourceProperties.getUrl()), 70 | Optional.ofNullable(this.flywayProperties.getUser()).orElse(dataSourceProperties.getUsername()), 71 | Optional.ofNullable(this.flywayProperties.getPassword()).orElse(dataSourceProperties.getPassword())); 72 | 73 | config.initSql(String.join(";\n", flywayProperties.getInitSqls())); 74 | } else if (this.flywayDataSource != null) { 75 | config.dataSource(this.flywayDataSource); 76 | } else { 77 | config.dataSource(dataSource); 78 | } 79 | 80 | config.locations("classpath:db_nakadiproducer/migrations"); 81 | config.schemas("nakadi_events"); 82 | if (callbacks != null) { 83 | config.callbacks(callbacks.stream().map(FlywayCallbackAdapter::new).toArray(FlywayCallbackAdapter[]::new)); 84 | } 85 | 86 | config.baselineOnMigrate(true); 87 | config.baselineVersion("2133546886.1.0"); 88 | 89 | Flyway flyway = new Flyway(config); 90 | flyway.migrate(); 91 | } 92 | 93 | private static class FlywayCallbackAdapter extends BaseCallback { 94 | 95 | private final NakadiProducerFlywayCallback callback; 96 | 97 | private final Set supportedCallbacks = Stream.of( 98 | BEFORE_CLEAN, 99 | AFTER_CLEAN, 100 | BEFORE_MIGRATE, 101 | BEFORE_EACH_MIGRATE, 102 | AFTER_EACH_MIGRATE, 103 | AFTER_MIGRATE, 104 | BEFORE_UNDO, 105 | BEFORE_EACH_UNDO, 106 | AFTER_EACH_UNDO, 107 | AFTER_UNDO, 108 | BEFORE_VALIDATE, 109 | AFTER_VALIDATE, 110 | BEFORE_BASELINE, 111 | AFTER_BASELINE, 112 | BEFORE_REPAIR, 113 | AFTER_REPAIR, 114 | BEFORE_INFO, 115 | AFTER_INFO 116 | 117 | ).collect(Collectors.toSet()); 118 | 119 | private FlywayCallbackAdapter(NakadiProducerFlywayCallback callback) { 120 | this.callback = callback; 121 | } 122 | 123 | @Override 124 | public boolean supports(Event event, Context context) { 125 | return supportedCallbacks.contains(event); 126 | } 127 | 128 | @Override 129 | public void handle(Event event, Context context) { 130 | switch (event) { 131 | case BEFORE_CLEAN: 132 | callback.beforeClean(context.getConnection()); 133 | break; 134 | case AFTER_CLEAN: 135 | callback.afterClean(context.getConnection()); 136 | break; 137 | case BEFORE_MIGRATE: 138 | callback.beforeMigrate(context.getConnection()); 139 | break; 140 | case BEFORE_EACH_MIGRATE: 141 | callback.beforeEachMigrate(context.getConnection(), context.getMigrationInfo()); 142 | break; 143 | case AFTER_EACH_MIGRATE: 144 | callback.afterEachMigrate(context.getConnection(), context.getMigrationInfo()); 145 | break; 146 | case AFTER_MIGRATE: 147 | callback.afterMigrate(context.getConnection()); 148 | break; 149 | case BEFORE_UNDO: 150 | callback.beforeUndo(context.getConnection()); 151 | break; 152 | case BEFORE_EACH_UNDO: 153 | callback.beforeEachUndo(context.getConnection(), context.getMigrationInfo()); 154 | break; 155 | case AFTER_EACH_UNDO: 156 | callback.afterEachUndo(context.getConnection(), context.getMigrationInfo()); 157 | break; 158 | case AFTER_UNDO: 159 | callback.afterUndo(context.getConnection()); 160 | break; 161 | case BEFORE_VALIDATE: 162 | callback.beforeValidate(context.getConnection()); 163 | break; 164 | case AFTER_VALIDATE: 165 | callback.afterValidate(context.getConnection()); 166 | break; 167 | case BEFORE_BASELINE: 168 | callback.beforeBaseline(context.getConnection()); 169 | break; 170 | case AFTER_BASELINE: 171 | callback.afterBaseline(context.getConnection()); 172 | break; 173 | case BEFORE_REPAIR: 174 | callback.beforeRepair(context.getConnection()); 175 | break; 176 | case AFTER_REPAIR: 177 | callback.afterRepair(context.getConnection()); 178 | break; 179 | case BEFORE_INFO: 180 | callback.beforeInfo(context.getConnection()); 181 | break; 182 | case AFTER_INFO: 183 | callback.afterInfo(context.getConnection()); 184 | break; 185 | default: 186 | throw new IllegalStateException("Unexpected value: " + event); 187 | } 188 | } 189 | } 190 | } 191 | --------------------------------------------------------------------------------