├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── build.gradle ├── gradle ├── libraries.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── install-thrift-0.9.3.sh ├── kafka-test-harness ├── build.gradle └── src │ └── main │ └── scala │ └── com │ └── airbnb │ └── spinaltap │ └── kafka │ ├── AbstractKafkaIntegrationTestHarness.scala │ ├── AbstractKafkaServerTestHarness.scala │ ├── AbstractZookeeperTestHarness.scala │ ├── EmbeddedZookeeper.scala │ └── TestUtils.scala ├── settings.gradle ├── spinaltap-common ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── airbnb │ │ ├── common │ │ └── metrics │ │ │ ├── Counter.java │ │ │ ├── DropwizardCounter.java │ │ │ ├── DropwizardHistogram.java │ │ │ ├── Histogram.java │ │ │ ├── TaggedMetricRegistry.java │ │ │ └── TaggedMetricRegistryFactory.java │ │ └── spinaltap │ │ └── common │ │ ├── config │ │ ├── DestinationConfiguration.java │ │ ├── SourceConfiguration.java │ │ └── TlsConfiguration.java │ │ ├── destination │ │ ├── AbstractDestination.java │ │ ├── BufferedDestination.java │ │ ├── Destination.java │ │ ├── DestinationBuilder.java │ │ ├── DestinationMetrics.java │ │ ├── DestinationPool.java │ │ └── ListenableDestination.java │ │ ├── exception │ │ ├── AttributeValueDeserializationException.java │ │ ├── ColumnDeserializationException.java │ │ ├── DestinationException.java │ │ ├── EntityDeserializationException.java │ │ ├── SourceException.java │ │ └── SpinaltapException.java │ │ ├── metrics │ │ └── SpinalTapMetrics.java │ │ ├── pipe │ │ ├── AbstractPipeFactory.java │ │ ├── Pipe.java │ │ ├── PipeManager.java │ │ └── PipeMetrics.java │ │ ├── source │ │ ├── AbstractDataStoreSource.java │ │ ├── AbstractSource.java │ │ ├── ListenableSource.java │ │ ├── MysqlSourceState.java │ │ ├── Source.java │ │ ├── SourceEvent.java │ │ ├── SourceMetrics.java │ │ └── SourceState.java │ │ ├── util │ │ ├── BatchMapper.java │ │ ├── ChainedFilter.java │ │ ├── ClassBasedMapper.java │ │ ├── ConcurrencyUtil.java │ │ ├── ErrorHandler.java │ │ ├── Filter.java │ │ ├── Joiner.java │ │ ├── JsonUtil.java │ │ ├── KeyProvider.java │ │ ├── Mapper.java │ │ ├── Repository.java │ │ ├── StateRepositoryFactory.java │ │ ├── ThreeWayJoiner.java │ │ ├── Validator.java │ │ └── ZookeeperRepository.java │ │ └── validator │ │ └── MutationOrderValidator.java │ └── test │ └── java │ └── com │ └── airbnb │ └── spinaltap │ └── common │ ├── destination │ ├── AbstractDestinationTest.java │ ├── BufferedDestinationTest.java │ ├── DestinationBuilderTest.java │ ├── DestinationPoolTest.java │ └── ListenableDestinationTest.java │ ├── pipe │ ├── PipeManagerTest.java │ └── PipeTest.java │ ├── source │ ├── AbstractSourceTest.java │ └── ListenableSourceTest.java │ ├── util │ ├── ChainedFilterTest.java │ └── ClassBasedMapperTest.java │ └── validator │ └── MutationOrderValidatorTest.java ├── spinaltap-kafka ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── airbnb │ │ └── spinaltap │ │ └── kafka │ │ ├── KafkaDestination.java │ │ ├── KafkaDestinationBuilder.java │ │ └── KafkaProducerConfiguration.java │ └── test │ └── java │ └── com │ └── airbnb │ └── spinaltap │ └── kafka │ └── KafkaDestinationTest.java ├── spinaltap-model ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── airbnb │ │ │ └── spinaltap │ │ │ ├── Mutation.java │ │ │ └── mysql │ │ │ ├── BinlogFilePos.java │ │ │ ├── DataSource.java │ │ │ ├── GtidSet.java │ │ │ ├── Transaction.java │ │ │ └── mutation │ │ │ ├── MysqlDeleteMutation.java │ │ │ ├── MysqlInsertMutation.java │ │ │ ├── MysqlMutation.java │ │ │ ├── MysqlMutationMetadata.java │ │ │ ├── MysqlUpdateMutation.java │ │ │ └── schema │ │ │ ├── Column.java │ │ │ ├── ColumnDataType.java │ │ │ ├── ColumnMetadata.java │ │ │ ├── PrimaryKey.java │ │ │ ├── Row.java │ │ │ └── Table.java │ └── thrift │ │ └── com │ │ └── airbnb │ │ └── jitney │ │ └── event │ │ └── spinaltap │ │ └── spinaltap_v1.thrift │ └── test │ └── java │ └── com │ └── airbnb │ └── spinaltap │ ├── GtidSetTest.java │ ├── MutationTest.java │ └── mysql │ ├── BinlogFilePosTest.java │ ├── mutation │ └── MysqlUpdateMutationTest.java │ └── schema │ └── RowTest.java ├── spinaltap-mysql ├── build.gradle └── src │ ├── main │ ├── antlr │ │ └── com │ │ │ └── airbnb │ │ │ └── spinaltap │ │ │ └── mysql │ │ │ └── schema │ │ │ └── MySQL.g4 │ └── java │ │ └── com │ │ └── airbnb │ │ └── spinaltap │ │ └── mysql │ │ ├── ColumnSerializationUtil.java │ │ ├── MysqlClient.java │ │ ├── MysqlDestinationMetrics.java │ │ ├── MysqlPipeFactory.java │ │ ├── MysqlSource.java │ │ ├── MysqlSourceFactory.java │ │ ├── MysqlSourceMetrics.java │ │ ├── StateHistory.java │ │ ├── StateRepository.java │ │ ├── TableCache.java │ │ ├── binlog_connector │ │ ├── BinaryLogConnectorEventMapper.java │ │ └── BinaryLogConnectorSource.java │ │ ├── config │ │ ├── AbstractMysqlConfiguration.java │ │ ├── MysqlConfiguration.java │ │ └── MysqlSchemaStoreConfiguration.java │ │ ├── event │ │ ├── BinlogEvent.java │ │ ├── DeleteEvent.java │ │ ├── GTIDEvent.java │ │ ├── QueryEvent.java │ │ ├── StartEvent.java │ │ ├── TableMapEvent.java │ │ ├── UpdateEvent.java │ │ ├── WriteEvent.java │ │ ├── XidEvent.java │ │ ├── filter │ │ │ ├── DuplicateFilter.java │ │ │ ├── EventTypeFilter.java │ │ │ ├── MysqlEventFilter.java │ │ │ └── TableFilter.java │ │ └── mapper │ │ │ ├── DeleteMutationMapper.java │ │ │ ├── GTIDMapper.java │ │ │ ├── InsertMutationMapper.java │ │ │ ├── MysqlMutationMapper.java │ │ │ ├── QueryMapper.java │ │ │ ├── StartMapper.java │ │ │ ├── TableMapMapper.java │ │ │ ├── UpdateMutationMapper.java │ │ │ └── XidMapper.java │ │ ├── exception │ │ └── InvalidBinlogPositionException.java │ │ ├── mutation │ │ ├── MysqlKeyProvider.java │ │ └── mapper │ │ │ ├── DeleteMutationMapper.java │ │ │ ├── InsertMutationMapper.java │ │ │ ├── ThriftMutationMapper.java │ │ │ └── UpdateMutationMapper.java │ │ ├── schema │ │ ├── MysqlColumn.java │ │ ├── MysqlSchemaArchiver.java │ │ ├── MysqlSchemaDatabase.java │ │ ├── MysqlSchemaManager.java │ │ ├── MysqlSchemaManagerFactory.java │ │ ├── MysqlSchemaReader.java │ │ ├── MysqlSchemaStore.java │ │ ├── MysqlSchemaUtil.java │ │ └── MysqlTableSchema.java │ │ └── validator │ │ ├── EventOrderValidator.java │ │ └── MutationSchemaValidator.java │ └── test │ └── java │ └── com │ └── airbnb │ └── spinaltap │ └── mysql │ ├── ColumnSerializationUtilTest.java │ ├── EventOrderValidatorTest.java │ ├── MysqlSourceTest.java │ ├── StateHistoryTest.java │ ├── StateRepositoryTest.java │ ├── TableCacheTest.java │ ├── binlog_connector │ └── BinaryLogConnectorEventMapperTest.java │ ├── event │ ├── filter │ │ └── MysqlEventFilterTest.java │ └── mapper │ │ └── MysqlMutationMapperTest.java │ ├── mutation │ └── MysqlKeyProviderTest.java │ ├── schema │ ├── MysqlColumnTest.java │ ├── MysqlSchemaDatabaseTest.java │ ├── MysqlSchemaManagerTest.java │ ├── MysqlSchemaStoreTest.java │ └── MysqlSchemaUtilTest.java │ ├── util │ └── JsonSerializationTest.java │ └── validator │ └── MutationSchemaValidatorTest.java └── spinaltap-standalone ├── build.gradle └── src └── main ├── java └── com │ └── airbnb │ └── spinaltap │ ├── SpinalTapStandaloneApp.java │ ├── SpinalTapStandaloneConfiguration.java │ └── ZookeeperRepositoryFactory.java └── resources ├── log4j2.yaml └── spinaltap-standalone-config-sample.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | **/build/ 3 | classes 4 | thrift-gen/ 5 | 6 | GIT_CHECKOUT_PATHS 7 | 8 | # Gradle 9 | .gradle/ 10 | **/.gradle/ 11 | **/.gradletasknamecache 12 | 13 | # VIM 14 | *.swp 15 | *~ 16 | 17 | # Eclipse 18 | workbench.xmi 19 | 20 | .idea/ 21 | *.iml 22 | *.ipr 23 | *.iws 24 | **/*.iml 25 | out/ 26 | .classpath 27 | **/.classpath 28 | .project 29 | **/.project 30 | .settings/ 31 | **/.settings/ 32 | # Ignore OS X Desktop Services store 33 | .DS_Store 34 | *.DS_Store 35 | **/.DS_Store 36 | .tddium* 37 | **/*~ 38 | 39 | # NetBeans 40 | .nb-gradle/ 41 | .solano* 42 | 43 | # Bazel 44 | WORKSPACE 45 | **/BUILD 46 | third_party/ 47 | /bazel-bazel 48 | /bazel-bin 49 | /bazel-genfiles 50 | /bazel-io_bazel 51 | /bazel-out 52 | /bazel-testlogs 53 | /bazel.iml 54 | 55 | # VSCode 56 | .vscode/ 57 | 58 | # Gradle Enterprise 59 | .build_scan_info.json 60 | 61 | # log files 62 | *.log 63 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | dist: bionic 4 | 5 | sudo: true 6 | 7 | before_install: 8 | - sudo apt-get -qq update 9 | - sudo apt-get install -y automake bison flex g++ git libboost-all-dev libevent-dev libssl1.0-dev libtool make pkg-config openjdk-8-jdk 10 | - ./install-thrift-0.9.3.sh 11 | - wget https://raw.githubusercontent.com/michaelklishin/jdk_switcher/master/jdk_switcher.sh 12 | - source jdk_switcher.sh 13 | - jdk_switcher use openjdk8 14 | 15 | install: 16 | - ./gradlew spotlessCheck 17 | - ./gradlew assemble 18 | 19 | script: 20 | - ./gradlew check 21 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing to SpinalTap 2 | 3 | ### General Guideline 4 | - Please fork this repository (https://github.com/airbnb/SpinalTap) and make changes on the fork. 5 | - Please include corresponding unit/functional/integration tests if you are committing a non-trivial change. 6 | - Please run the `gradle spotlessApply` in the root directory to follow the code style. 7 | - Please do NOT change any public APIs without discussing with the team. 8 | - Please follow the `Design Template` to document the new feature design, and send a PR for review. 9 | 10 | ### Design Template 11 | - Motivation 12 | [Discuss the existing pain points and the advantages will get through the new features.] 13 | - Public Interface 14 | [Discuss what are the changes on the public APIs (newly added, deprecated, and deleted APIs).] 15 | - Proposed Changes 16 | [Discuss in details about the proposed changes.] 17 | - Compatibility, Deprecation, and Migration Plan 18 | [Discuss the potential compatibility issues and any migration process we need to follow if there is any.] 19 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | apply from: 'gradle/libraries.gradle' 2 | buildscript { 3 | repositories { 4 | maven { 5 | url "https://plugins.gradle.org/m2/" 6 | } 7 | } 8 | dependencies { 9 | classpath "com.diffplug.spotless:spotless-plugin-gradle:3.8.0" 10 | } 11 | } 12 | 13 | subprojects { 14 | apply plugin: 'java' 15 | apply plugin: 'com.diffplug.gradle.spotless' 16 | repositories { 17 | mavenCentral() 18 | } 19 | 20 | compileJava { 21 | sourceCompatibility = '1.8' 22 | targetCompatibility = '1.8' 23 | 24 | options.compilerArgs << '-Xlint' 25 | } 26 | 27 | spotless { 28 | enforceCheck = false 29 | java { 30 | licenseHeader '/** Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license information. */' 31 | googleJavaFormat('1.4') 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /gradle/libraries.gradle: -------------------------------------------------------------------------------- 1 | ext { 2 | libraries = [ 3 | antlr4: 'org.antlr:antlr4:4.7.1', 4 | apache_commons_lang: 'org.apache.commons:commons-lang3:3.7', 5 | apache_curator_framework: 'org.apache.curator:curator-framework:4.0.1', 6 | codahale_metrics_core: 'com.codahale.metrics:metrics-core:3.0.2', 7 | findbugs_jsr305: 'com.google.code.findbugs:jsr305:3.0.0', 8 | guava: 'com.google.guava:guava:22.0', 9 | guava_retrying: 'com.github.rholder:guava-retrying:1.0.6', 10 | hibernate_validator: 'org.hibernate:hibernate-validator:5.1.3.Final', 11 | icu4j: 'com.ibm.icu:icu4j:61.1', 12 | jackson_annotations: 'com.fasterxml.jackson.core:jackson-annotations:2.9.5', 13 | jackson_databind: 'com.fasterxml.jackson.core:jackson-databind:2.9.5', 14 | jackson_dataformat_yaml: 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.5', 15 | jackson_datatype_joda: 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.9.5', 16 | jackson_datatype_guava: 'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.9.5', 17 | javax_validaton: 'javax.validation:validation-api:1.1.0.Final', 18 | jdbi3: 'org.jdbi:jdbi3-core:3.10.1', 19 | junit: 'junit:junit:4.12', 20 | kafka_core: 'org.apache.kafka:kafka_2.10:0.9.0.1', 21 | kafka_clients: 'org.apache.kafka:kafka-clients:0.9.0.1', 22 | kafka_test: 'org.apache.kafka:kafka_2.10:0.9.0.1:test', 23 | log4j_slf4j_impl: 'org.apache.logging.log4j:log4j-slf4j-impl:2.9.1', 24 | lombok: 'org.projectlombok:lombok:1.18.2', 25 | mokito: 'org.mockito:mockito-all:1.10.19', 26 | mysql_binlog_connector: 'com.github.shyiko:mysql-binlog-connector-java:0.20.1', 27 | mysql_connector: 'mysql:mysql-connector-java:5.1.44', 28 | slf4j: 'org.slf4j:slf4j-api:1.7.25', 29 | thrift: 'org.apache.thrift:libthrift:0.9.3', 30 | zookeeper: 'org.apache.zookeeper:zookeeper:3.4.11' 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/airbnb/SpinalTap/6992bc3352a9ecf74ca40ec93c9c56bbbc5dc413/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /install-thrift-0.9.3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -ex 3 | wget http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz 4 | tar xzf thrift-0.9.3.tar.gz 5 | cd thrift-0.9.3 6 | ./configure --without-qt4 --without-qt5 --without-csharp --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-dart --without-ruby --without-haskell --without-go --without-haxe --without-d 7 | sed -i -e 's#mvn.repo=http://repo1.maven.org/maven2#mvn.repo=https://repo1.maven.org/maven2#g' ./lib/java/build.properties 8 | make 9 | sudo make install 10 | -------------------------------------------------------------------------------- /kafka-test-harness/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'scala' 3 | 4 | dependencies { 5 | compile libraries.kafka_core 6 | compile libraries.kafka_clients 7 | compile libraries.kafka_test 8 | } 9 | -------------------------------------------------------------------------------- /kafka-test-harness/src/main/scala/com/airbnb/spinaltap/kafka/AbstractKafkaIntegrationTestHarness.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License").
 See License in the project root for license information. 3 | */ 4 | 5 | package com.airbnb.spinaltap.kafka 6 | 7 | import java.util.Properties 8 | 9 | import kafka.server.KafkaConfig 10 | 11 | /** 12 | * LinkedIn integration test harness for Kafka 13 | * This is simply a copy of open source code, we do this because java does not support trait, we are making it abstract 14 | * class so user java test class can extend it. 15 | */ 16 | abstract class AbstractKafkaIntegrationTestHarness extends AbstractKafkaServerTestHarness { 17 | 18 | def generateConfigs() = 19 | TestUtils.createBrokerConfigs(clusterSize(), zkConnect, enableControlledShutdown = false).map(KafkaConfig.fromProps(_, overridingProps())) 20 | 21 | /** 22 | * User can override this method to return the number of brokers they want. 23 | * By default only one broker will be launched. 24 | * @return the number of brokers needed in the Kafka cluster for the test. 25 | */ 26 | def clusterSize(): Int = 1 27 | 28 | /** 29 | * User can override this method to apply customized configurations to the brokers. 30 | * By default the only configuration is number of partitions when topics get automatically created. The default value 31 | * is 1. 32 | * @return The configurations to be used by brokers. 33 | */ 34 | def overridingProps(): Properties = { 35 | val props = new Properties() 36 | props.setProperty(KafkaConfig.NumPartitionsProp, 1.toString) 37 | props 38 | } 39 | 40 | /** 41 | * Returns the bootstrap servers configuration string to be used by clients. 42 | * @return bootstrap servers string. 43 | */ 44 | def bootstrapServers(): String = super.bootstrapUrl 45 | 46 | /** 47 | * This method should be defined as @beforeMethod. 48 | */ 49 | override def setUp() { 50 | super.setUp() 51 | } 52 | 53 | /** 54 | * This method should be defined as @AfterMethod. 55 | */ 56 | override def tearDown() { 57 | super.tearDown() 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /kafka-test-harness/src/main/scala/com/airbnb/spinaltap/kafka/AbstractKafkaServerTestHarness.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License").
 See License in the project root for license information. 3 | */ 4 | 5 | package com.airbnb.spinaltap.kafka 6 | 7 | import java.io.File 8 | import java.util 9 | import java.util.Properties 10 | 11 | import kafka.common.KafkaException 12 | import kafka.server.{KafkaConfig, KafkaServer} 13 | import kafka.utils.CoreUtils 14 | import org.apache.kafka.common.protocol.SecurityProtocol 15 | import org.apache.kafka.common.security.auth.KafkaPrincipal 16 | 17 | import scala.collection.mutable 18 | 19 | /** 20 | * Kafka server integration test harness. 21 | * This is simply a copy of open source code, we do this because java does not support trait, we are making it abstract 22 | * class so user java test class can extend it. 23 | */ 24 | abstract class AbstractKafkaServerTestHarness extends AbstractZookeeperTestHarness { 25 | var instanceConfigs: Seq[KafkaConfig] = null 26 | var servers: mutable.Buffer[KafkaServer] = null 27 | var brokerList: String = null 28 | var alive: Array[Boolean] = null 29 | val kafkaPrincipalType = KafkaPrincipal.USER_TYPE 30 | val setClusterAcl: Option[() => Unit] = None 31 | 32 | /** 33 | * Implementations must override this method to return a set of KafkaConfigs. This method will be invoked for every 34 | * test and should not reuse previous configurations unless they select their ports randomly when servers are started. 35 | */ 36 | def generateConfigs(): Seq[KafkaConfig] 37 | 38 | def serverForId(id: Int) = servers.find(s => s.config.brokerId == id) 39 | 40 | def bootstrapUrl: String = brokerList 41 | 42 | protected def securityProtocol: SecurityProtocol = SecurityProtocol.PLAINTEXT 43 | 44 | protected def trustStoreFile: Option[File] = None 45 | 46 | protected def saslProperties: Option[Properties] = None 47 | 48 | override def setUp() { 49 | super.setUp() 50 | val configs = generateConfigs() 51 | if (configs.size <= 0) 52 | throw new KafkaException("Must supply at least one server config.") 53 | servers = configs.map(TestUtils.createServer(_)).toBuffer 54 | brokerList = TestUtils.getBrokerListStrFromServers(servers, securityProtocol) 55 | alive = new Array[Boolean](servers.length) 56 | util.Arrays.fill(alive, true) 57 | // We need to set a cluster ACL in some cases here 58 | // because of the topic creation in the setup of 59 | // IntegrationTestHarness. If we don't, then tests 60 | // fail with a cluster action authorization exception 61 | // when processing an update metadata request 62 | // (controller -> broker). 63 | // 64 | // The following method does nothing by default, but 65 | // if the test case requires setting up a cluster ACL, 66 | // then it needs to be implemented. 67 | setClusterAcl.foreach(_.apply()) 68 | } 69 | 70 | override def tearDown() { 71 | if (servers != null) { 72 | servers.foreach(_.shutdown()) 73 | servers.foreach(server => CoreUtils.rm(server.config.logDirs)) 74 | } 75 | super.tearDown() 76 | } 77 | 78 | /** 79 | * Pick a broker at random and kill it if it isn't already dead 80 | * Return the id of the broker killed 81 | */ 82 | def killRandomBroker(): Int = { 83 | val index = TestUtils.random.nextInt(servers.length) 84 | if (alive(index)) { 85 | servers(index).shutdown() 86 | servers(index).awaitShutdown() 87 | alive(index) = false 88 | } 89 | index 90 | } 91 | 92 | /** 93 | * Restart any dead brokers 94 | */ 95 | def restartDeadBrokers() { 96 | for (i <- 0 until servers.length if !alive(i)) { 97 | servers(i).startup() 98 | alive(i) = true 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /kafka-test-harness/src/main/scala/com/airbnb/spinaltap/kafka/AbstractZookeeperTestHarness.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License").
 See License in the project root for license information. 3 | */ 4 | 5 | package com.airbnb.spinaltap.kafka 6 | 7 | import javax.security.auth.login.Configuration 8 | 9 | import kafka.utils.{CoreUtils, Logging, ZkUtils} 10 | import org.apache.kafka.common.security.JaasUtils 11 | import java.io.IOException 12 | import java.net.{SocketTimeoutException, Socket, InetAddress, InetSocketAddress} 13 | 14 | /** 15 | * Zookeeper test harness. 16 | * This is simply a copy of open source code, we do this because java does not support trait, we are making it abstract 17 | * class so user java test class can extend it. 18 | */ 19 | abstract class AbstractZookeeperTestHarness extends Logging { 20 | 21 | val zkConnectionTimeout = 6000 22 | val zkSessionTimeout = 6000 23 | 24 | var zkUtils: ZkUtils = null 25 | var zookeeper: EmbeddedZookeeper = null 26 | 27 | def zkPort: Int = zookeeper.port 28 | 29 | def zkConnect: String = s"127.0.0.1:$zkPort" 30 | 31 | def setUp() { 32 | zookeeper = new EmbeddedZookeeper() 33 | zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, JaasUtils.isZkSecurityEnabled) 34 | } 35 | 36 | def tearDown() { 37 | if (zkUtils != null) 38 | CoreUtils.swallow(zkUtils.close()) 39 | if (zookeeper != null) 40 | CoreUtils.swallow(zookeeper.shutdown()) 41 | 42 | def isDown: Boolean = { 43 | try { 44 | sendStat("127.0.0.1", zkPort, 3000) 45 | false 46 | } catch { 47 | case _: Throwable => 48 | debug("Server is down") 49 | true 50 | } 51 | } 52 | 53 | Iterator.continually(isDown).exists(identity) 54 | 55 | Configuration.setConfiguration(null) 56 | } 57 | 58 | /** 59 | * Copied from kafka.zk.ZkFourLetterWords.scala which is NOT included in the kafka-clients version 60 | * we are using. 61 | */ 62 | def sendStat(host: String, port: Int, timeout: Int) { 63 | val hostAddress = 64 | if (host != null) new InetSocketAddress(host, port) 65 | else new InetSocketAddress(InetAddress.getByName(null), port) 66 | val sock = new Socket() 67 | try { 68 | sock.connect(hostAddress, timeout) 69 | val outStream = sock.getOutputStream 70 | outStream.write("stat".getBytes) 71 | outStream.flush() 72 | } catch { 73 | case e: SocketTimeoutException => throw new IOException("Exception while sending 4lw") 74 | } finally { 75 | sock.close 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /kafka-test-harness/src/main/scala/com/airbnb/spinaltap/kafka/EmbeddedZookeeper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License").
 See License in the project root for license information. 3 | */ 4 | 5 | package com.airbnb.spinaltap.kafka 6 | 7 | import java.net.InetSocketAddress 8 | 9 | import kafka.utils.CoreUtils 10 | import org.apache.kafka.common.utils.Utils 11 | import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer} 12 | 13 | /** 14 | * This is a copy of open source Kafka embedded zookeeper class. We put it here to avoid dependency on o.a.k.test jar. 15 | */ 16 | class EmbeddedZookeeper() { 17 | val snapshotDir = TestUtils.tempDir() 18 | val logDir = TestUtils.tempDir() 19 | val tickTime = 500 20 | val zookeeper = new ZooKeeperServer(snapshotDir, logDir, tickTime) 21 | val factory = new NIOServerCnxnFactory() 22 | private val addr = new InetSocketAddress("127.0.0.1", TestUtils.RandomPort) 23 | factory.configure(addr, 0) 24 | factory.startup(zookeeper) 25 | val port = zookeeper.getClientPort 26 | 27 | def shutdown() { 28 | CoreUtils.swallow(zookeeper.shutdown()) 29 | CoreUtils.swallow(factory.shutdown()) 30 | Utils.delete(logDir) 31 | Utils.delete(snapshotDir) 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include "spinaltap-common" 2 | include "spinaltap-kafka" 3 | include "spinaltap-model" 4 | include "spinaltap-mysql" 5 | include "spinaltap-standalone" 6 | include "kafka-test-harness" -------------------------------------------------------------------------------- /spinaltap-common/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile project(':spinaltap-model') 3 | compile libraries.apache_commons_lang 4 | compile libraries.codahale_metrics_core 5 | compile libraries.findbugs_jsr305 6 | compile libraries.icu4j 7 | compile libraries.jackson_databind 8 | compile libraries.jackson_datatype_joda 9 | compile libraries.javax_validaton 10 | compile libraries.log4j_slf4j_impl 11 | compile libraries.zookeeper 12 | compileOnly libraries.lombok 13 | annotationProcessor libraries.lombok 14 | 15 | // Running in ZooKeeper 3.4.x compatibility mode 16 | compile(libraries.apache_curator_framework) { 17 | exclude group: 'org.apache.zookeeper', module: 'zookeeper' 18 | } 19 | 20 | testCompile libraries.junit 21 | testCompile libraries.mokito 22 | testCompileOnly libraries.lombok 23 | testAnnotationProcessor libraries.lombok 24 | } 25 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/common/metrics/Counter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.common.metrics; 6 | 7 | /** A monotone counter */ 8 | public interface Counter { 9 | /** Increment the counter by one. */ 10 | public void inc(); 11 | 12 | /** 13 | * Increment the counter by {@code n}. 14 | * 15 | * @param n the amount by which the counter will be increased 16 | */ 17 | public void inc(long n); 18 | } 19 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/common/metrics/DropwizardCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.common.metrics; 6 | 7 | /** 8 | * Our own version of the Counter class that multiplexes tagged metrics to tagged and non-tagged 9 | * versions, because Datadog is not capable of averaging across tag values correctly. 10 | */ 11 | public class DropwizardCounter implements Counter { 12 | 13 | private final com.codahale.metrics.Counter[] counters; 14 | 15 | public DropwizardCounter(com.codahale.metrics.Counter... counters) { 16 | this.counters = counters; 17 | } 18 | 19 | /** Increment the counter by one. */ 20 | public void inc() { 21 | inc(1); 22 | } 23 | 24 | /** 25 | * Increment the counter by {@code n}. 26 | * 27 | * @param n the amount by which the counter will be increased 28 | */ 29 | public void inc(long n) { 30 | for (com.codahale.metrics.Counter counter : counters) { 31 | counter.inc(n); 32 | } 33 | } 34 | 35 | /** Decrement the counter by one. */ 36 | public void dec() { 37 | dec(1); 38 | } 39 | 40 | /** 41 | * Decrement the counter by {@code n}. 42 | * 43 | * @param n the amount by which the counter will be decreased 44 | */ 45 | public void dec(long n) { 46 | for (com.codahale.metrics.Counter counter : counters) { 47 | counter.dec(n); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/common/metrics/DropwizardHistogram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.common.metrics; 6 | 7 | /** 8 | * Our own version of the Histogram class that multiplexes tagged metrics to tagged and non-tagged 9 | * versions, because Datadog is not capable of averaging across tag values correctly. 10 | */ 11 | public class DropwizardHistogram implements Histogram { 12 | 13 | private final com.codahale.metrics.Histogram[] histograms; 14 | 15 | public DropwizardHistogram(com.codahale.metrics.Histogram... histograms) { 16 | this.histograms = histograms; 17 | } 18 | 19 | /** 20 | * Adds a recorded value. 21 | * 22 | * @param value the length of the value 23 | */ 24 | public void update(int value) { 25 | update((long) value); 26 | } 27 | 28 | /** 29 | * Adds a recorded value. 30 | * 31 | * @param value the length of the value 32 | */ 33 | public void update(long value) { 34 | for (com.codahale.metrics.Histogram histogram : histograms) { 35 | histogram.update(value); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/common/metrics/Histogram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.common.metrics; 6 | 7 | /** A Histogram interface used to record and analyze the distribution of values. */ 8 | public interface Histogram { 9 | /** 10 | * Adds a recorded value. 11 | * 12 | * @param value the length of the value 13 | */ 14 | public void update(long value); 15 | } 16 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/common/metrics/TaggedMetricRegistryFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.common.metrics; 6 | 7 | import lombok.NonNull; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | @Slf4j 11 | /** 12 | * This provides an easy way to get the TaggedMetricRegistry in any class. The initialize must be 13 | * called before getting the TaggedMetricRegistry, otherwise the metrics will be lost. 14 | * 15 | *

The TaggedMetricRegistry is also available for injection. Consider injecting it directly in 16 | * case you want metrics during object initialization or as a general dependency injection best 17 | * practice. 18 | */ 19 | public class TaggedMetricRegistryFactory { 20 | 21 | private static volatile TaggedMetricRegistry registry = 22 | TaggedMetricRegistry.NON_INITIALIZED_TAGGED_METRIC_REGISTRY; 23 | 24 | private TaggedMetricRegistryFactory() {} 25 | 26 | public static void initialize(@NonNull TaggedMetricRegistry taggedMetricRegistry) { 27 | registry = taggedMetricRegistry; 28 | } 29 | 30 | public static TaggedMetricRegistry get() { 31 | if (registry == TaggedMetricRegistry.NON_INITIALIZED_TAGGED_METRIC_REGISTRY) { 32 | log.warn( 33 | "get() called before metrics is initialized. return NON_INITIALIZED_TAGGED_METRIC_REGISTRY."); 34 | } 35 | return registry; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/config/DestinationConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.config; 6 | 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import java.util.Map; 9 | import javax.validation.constraints.Min; 10 | import lombok.Data; 11 | import lombok.NoArgsConstructor; 12 | import lombok.NonNull; 13 | 14 | /** Represents a {@link com.airbnb.spinaltap.common.destination.Destination} configuration. */ 15 | @Data 16 | @NoArgsConstructor 17 | public class DestinationConfiguration { 18 | public static final String DEFAULT_TYPE = "kafka"; 19 | public static final int DEFAULT_BUFFER_SIZE = 0; 20 | public static final int DEFAULT_POOL_SIZE = 0; 21 | 22 | /** The destination type. Default to "kafka". */ 23 | @JsonProperty("type") 24 | @NonNull 25 | private String type = DEFAULT_TYPE; 26 | 27 | /** 28 | * The buffer size. If greater than 0, a {@link 29 | * com.airbnb.spinaltap.common.destination.BufferedDestination} will be constructed. 30 | */ 31 | @Min(0) 32 | @JsonProperty("buffer_size") 33 | private int bufferSize = DEFAULT_BUFFER_SIZE; 34 | 35 | /** 36 | * The pool size. If greater than 0, a {@link 37 | * com.airbnb.spinaltap.common.destination.DestinationPool} will be constructed with the specified 38 | * number of {@link com.airbnb.spinaltap.common.destination.Destination}s. 39 | */ 40 | @Min(0) 41 | @JsonProperty("pool_size") 42 | private int poolSize = DEFAULT_POOL_SIZE; 43 | 44 | @JsonProperty("producer_config") 45 | private Map producerConfig; 46 | } 47 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/config/SourceConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.config; 6 | 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import javax.validation.constraints.Min; 9 | import javax.validation.constraints.NotNull; 10 | import lombok.AllArgsConstructor; 11 | import lombok.Data; 12 | import lombok.NoArgsConstructor; 13 | import lombok.NonNull; 14 | 15 | /** Represents a {@code Source} configuration. */ 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class SourceConfiguration { 20 | private static int DEFAULT_REPLICAS = 3; 21 | private static int DEFAULT_PARTITIONS = 1; 22 | 23 | public SourceConfiguration( 24 | @NonNull final String name, 25 | final String type, 26 | final String instanceTag, 27 | @NonNull final DestinationConfiguration destinationConfiguration) { 28 | this.name = name; 29 | this.type = type; 30 | this.instanceGroupTag = instanceTag; 31 | this.destinationConfiguration = destinationConfiguration; 32 | } 33 | 34 | public SourceConfiguration(String type, String instanceTag) { 35 | this.type = type; 36 | this.instanceGroupTag = instanceTag; 37 | } 38 | 39 | /** The source name. */ 40 | @NotNull @JsonProperty private String name; 41 | 42 | /** 43 | * The number of replicas to stream from the source. 44 | * 45 | *

Note: This is only applicable if a cluster solution is employed. A Master-Replica state 46 | * transition model is recommended, where one cluster instance (master) is streaming events from a 47 | * given source at any point in time. This is required to ensure ordering guarantees. Replicas 48 | * will be promoted to Master in case of failure. Increasing number of replicas can be used to 49 | * improve fault tolerance. 50 | */ 51 | @Min(1) 52 | @JsonProperty 53 | private int replicas = DEFAULT_REPLICAS; 54 | 55 | /** 56 | * The number of stream partitions for a given source. 57 | * 58 | *

Note: This is only applicable if a cluster solution is employed. 59 | */ 60 | @Min(1) 61 | @JsonProperty 62 | private int partitions = DEFAULT_PARTITIONS; 63 | 64 | /** The source type (ex: MySQL, DynamoDB) */ 65 | @JsonProperty("type") 66 | private String type; 67 | 68 | /** 69 | * The group tag for cluster instances of the given source. 70 | * 71 | *

Note: This is only applicable if a cluster solution is employed. Tagging is used to indicate 72 | * the instances streaming a particular source. 73 | */ 74 | @JsonProperty("instance_group_tag") 75 | private String instanceGroupTag; 76 | 77 | /** The destination configuration for the specified source. */ 78 | @JsonProperty("destination") 79 | private DestinationConfiguration destinationConfiguration = new DestinationConfiguration(); 80 | } 81 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/config/TlsConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.config; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | import java.io.FileInputStream; 10 | import java.security.KeyStore; 11 | import javax.net.ssl.KeyManager; 12 | import javax.net.ssl.KeyManagerFactory; 13 | import javax.net.ssl.TrustManager; 14 | import javax.net.ssl.TrustManagerFactory; 15 | import lombok.AllArgsConstructor; 16 | import lombok.Data; 17 | import lombok.NoArgsConstructor; 18 | 19 | @Data 20 | @AllArgsConstructor 21 | @NoArgsConstructor 22 | @JsonIgnoreProperties(ignoreUnknown = true) 23 | public class TlsConfiguration { 24 | @JsonProperty("key_store_file_path") 25 | private String keyStoreFilePath; 26 | 27 | @JsonProperty("key_store_password") 28 | private String keyStorePassword; 29 | 30 | @JsonProperty("key_store_type") 31 | private String keyStoreType; 32 | 33 | @JsonProperty("trust_store_file_path") 34 | private String trustStoreFilePath; 35 | 36 | @JsonProperty("trust_store_password") 37 | private String trustStorePassword; 38 | 39 | @JsonProperty("trust_store_type") 40 | private String trustStoreType; 41 | 42 | public KeyManagerFactory getKeyManagerFactory() throws Exception { 43 | if (keyStoreFilePath != null && keyStorePassword != null) { 44 | KeyStore keyStore = 45 | KeyStore.getInstance(keyStoreType == null ? KeyStore.getDefaultType() : keyStoreType); 46 | keyStore.load(new FileInputStream(keyStoreFilePath), keyStorePassword.toCharArray()); 47 | KeyManagerFactory keyManagerFactory = 48 | KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); 49 | keyManagerFactory.init(keyStore, keyStorePassword.toCharArray()); 50 | return keyManagerFactory; 51 | } 52 | return null; 53 | } 54 | 55 | public KeyManager[] getKeyManagers() throws Exception { 56 | KeyManagerFactory keyManagerFactory = getKeyManagerFactory(); 57 | return keyManagerFactory == null ? null : keyManagerFactory.getKeyManagers(); 58 | } 59 | 60 | public TrustManagerFactory getTrustManagerFactory() throws Exception { 61 | if (trustStoreFilePath != null && trustStorePassword != null) { 62 | KeyStore keyStore = 63 | KeyStore.getInstance(trustStoreType == null ? KeyStore.getDefaultType() : trustStoreType); 64 | keyStore.load(new FileInputStream(trustStoreFilePath), trustStorePassword.toCharArray()); 65 | TrustManagerFactory trustManagerFactory = 66 | TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); 67 | trustManagerFactory.init(keyStore); 68 | return trustManagerFactory; 69 | } 70 | return null; 71 | } 72 | 73 | public TrustManager[] getTrustManagers() throws Exception { 74 | TrustManagerFactory trustManagerFactory = getTrustManagerFactory(); 75 | return trustManagerFactory == null ? null : trustManagerFactory.getTrustManagers(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/destination/AbstractDestination.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.exception.DestinationException; 9 | import com.airbnb.spinaltap.common.util.BatchMapper; 10 | import com.google.common.base.Stopwatch; 11 | import java.util.List; 12 | import java.util.concurrent.TimeUnit; 13 | import java.util.concurrent.atomic.AtomicBoolean; 14 | import java.util.concurrent.atomic.AtomicReference; 15 | import java.util.stream.Collectors; 16 | import lombok.NonNull; 17 | import lombok.RequiredArgsConstructor; 18 | import lombok.extern.slf4j.Slf4j; 19 | 20 | @Slf4j 21 | @RequiredArgsConstructor 22 | public abstract class AbstractDestination extends ListenableDestination { 23 | @NonNull private final BatchMapper, T> mapper; 24 | @NonNull private final DestinationMetrics metrics; 25 | private final long delaySendMs; 26 | 27 | private final AtomicBoolean started = new AtomicBoolean(false); 28 | private final AtomicReference> lastPublishedMutation = new AtomicReference<>(); 29 | 30 | @Override 31 | public Mutation getLastPublishedMutation() { 32 | return lastPublishedMutation.get(); 33 | } 34 | 35 | @SuppressWarnings("unchecked") 36 | @Override 37 | public void send(@NonNull final List> mutations) { 38 | if (mutations.isEmpty()) { 39 | return; 40 | } 41 | 42 | try { 43 | final Stopwatch stopwatch = Stopwatch.createStarted(); 44 | 45 | // introduce delay before mapper apply 46 | final Mutation latestMutation = mutations.get(mutations.size() - 1); 47 | delay(latestMutation); 48 | 49 | final List messages = mapper.apply(mutations.stream().collect(Collectors.toList())); 50 | publish(messages); 51 | 52 | lastPublishedMutation.set(latestMutation); 53 | 54 | stopwatch.stop(); 55 | final long time = stopwatch.elapsed(TimeUnit.MILLISECONDS); 56 | 57 | metrics.publishTime(time); 58 | metrics.publishSucceeded(mutations); 59 | 60 | log(mutations); 61 | notifySend(mutations); 62 | 63 | } catch (Exception ex) { 64 | log.error("Failed to send {} mutations.", mutations.size(), ex); 65 | mutations.forEach(mutation -> metrics.publishFailed(mutation, ex)); 66 | 67 | throw new DestinationException("Failed to send mutations", ex); 68 | } 69 | } 70 | 71 | /** 72 | * Induces a delay given the configured delay time. 73 | * 74 | * @param mutation The {@link Mutation} for which to consider the delay 75 | * @throws InterruptedException 76 | */ 77 | private void delay(final Mutation mutation) throws InterruptedException { 78 | final long delayMs = System.currentTimeMillis() - mutation.getMetadata().getTimestamp(); 79 | if (delayMs >= delaySendMs) { 80 | return; 81 | } 82 | 83 | Thread.sleep(delaySendMs - delayMs); 84 | } 85 | 86 | public abstract void publish(List messages) throws Exception; 87 | 88 | private void log(final List> mutations) { 89 | mutations.forEach( 90 | mutation -> 91 | log.trace( 92 | "Sent {} mutations with metadata {}.", mutation.getType(), mutation.getMetadata())); 93 | } 94 | 95 | @Override 96 | public boolean isStarted() { 97 | return started.get(); 98 | } 99 | 100 | @Override 101 | public void open() { 102 | lastPublishedMutation.set(null); 103 | super.open(); 104 | 105 | started.set(true); 106 | } 107 | 108 | @Override 109 | public void close() { 110 | started.set(false); 111 | } 112 | 113 | @Override 114 | public void clear() { 115 | metrics.clear(); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/destination/Destination.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | /** 12 | * Represents the receiving end of the {@link com.airbnb.spinaltap.common.pipe.Pipe}, where {@link 13 | * Mutation}s are published, i.e. a Sink. 14 | */ 15 | public interface Destination { 16 | /** @return the last {@link Mutation} that has been successfully published. */ 17 | Mutation getLastPublishedMutation(); 18 | 19 | default void send(Mutation mutation) { 20 | send(Collections.singletonList(mutation)); 21 | } 22 | 23 | /** 24 | * Publishes a list of {@link Mutation}s. 25 | * 26 | *

Note: On failure, streaming should be halted and the error propagated to avoid potential. 27 | * event loss 28 | */ 29 | void send(List> mutations); 30 | 31 | /** Adds a {@link Listener} to the destination. */ 32 | void addListener(Listener listener); 33 | 34 | /** Removes a {@link Listener} from the destination. */ 35 | void removeListener(Listener listener); 36 | 37 | /** @return whether the destination is started and publishing {@link Mutation}s */ 38 | boolean isStarted(); 39 | 40 | /** 41 | * Initializes the destination and prepares for {@link Mutation} publishing. 42 | * 43 | *

The operation should be idempotent. 44 | */ 45 | void open(); 46 | 47 | /** 48 | * Stops {@link Mutation} publishing and closes the destination. 49 | * 50 | *

The operation should be idempotent. 51 | */ 52 | void close(); 53 | 54 | /** 55 | * Clears the state of the destination. 56 | * 57 | *

The operation should be idempotent. 58 | */ 59 | void clear(); 60 | 61 | /** Represents a destination listener to get notified of events and lifecycle changes. */ 62 | abstract class Listener { 63 | /** Action to perform after the {@link Destination} has started. */ 64 | public void onStart() {} 65 | 66 | /** Action to perform after a list of {@link Mutation}s has been published. */ 67 | public void onSend(List> mutations) {} 68 | 69 | /** Action to perform when an error is caught on send. */ 70 | public void onError(Exception ex) {} 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/destination/ListenableDestination.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import lombok.NonNull; 11 | 12 | /** 13 | * Base {@link Destination} implement using observer pattern to allow listening to 15 | * streamed events and subscribe to lifecycle change notifications. 16 | */ 17 | abstract class ListenableDestination implements Destination { 18 | private final List listeners = new ArrayList<>(); 19 | 20 | @Override 21 | public void addListener(@NonNull final Listener listener) { 22 | listeners.add(listener); 23 | } 24 | 25 | @Override 26 | public void removeListener(@NonNull final Listener listener) { 27 | listeners.remove(listener); 28 | } 29 | 30 | protected void notifyStart() { 31 | listeners.forEach(Destination.Listener::onStart); 32 | } 33 | 34 | protected void notifySend(final List> mutations) { 35 | listeners.forEach(listener -> listener.onSend(mutations)); 36 | } 37 | 38 | protected void notifyError(final Exception ex) { 39 | listeners.forEach(listener -> listener.onError(ex)); 40 | } 41 | 42 | @Override 43 | public void open() { 44 | notifyStart(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/AttributeValueDeserializationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | /** Exception thrown when a DynamoDB attribute value cannot be deserialized. */ 8 | public final class AttributeValueDeserializationException extends EntityDeserializationException { 9 | private static final long serialVersionUID = 2442564527939878665L; 10 | 11 | private static String createMessage(String attributeName, String tableName) { 12 | return String.format( 13 | "Could not deserialize thrift bytebuffer for DynamoDB attribute %s in table %s.", 14 | attributeName, tableName); 15 | } 16 | 17 | public AttributeValueDeserializationException( 18 | final String attributeName, final String tableName, final Throwable cause) { 19 | super(createMessage(attributeName, tableName), cause); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/ColumnDeserializationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | /** Exception thrown when a MySQL column value cannot be deserialized. */ 8 | public final class ColumnDeserializationException extends EntityDeserializationException { 9 | private static final long serialVersionUID = 935990977706712032L; 10 | 11 | private static String createMessage(String columnName, String tableName) { 12 | return String.format( 13 | "Failed to deserialize MySQL column %s in table %s", columnName, tableName); 14 | } 15 | 16 | public ColumnDeserializationException( 17 | final String columnName, final String tableName, final Throwable cause) { 18 | super(createMessage(columnName, tableName), cause); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/DestinationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | public class DestinationException extends SpinaltapException { 8 | private static final long serialVersionUID = -2160287795842968357L; 9 | 10 | public DestinationException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/EntityDeserializationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | public class EntityDeserializationException extends SpinaltapException { 8 | private static final long serialVersionUID = 2604256281318886726L; 9 | 10 | public EntityDeserializationException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/SourceException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | public class SourceException extends SpinaltapException { 8 | private static final long serialVersionUID = -59599391802331914L; 9 | 10 | public SourceException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/exception/SpinaltapException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.exception; 6 | 7 | public class SpinaltapException extends RuntimeException { 8 | private static final long serialVersionUID = -8074916613284028245L; 9 | 10 | public SpinaltapException(String message) { 11 | super(message); 12 | } 13 | 14 | public SpinaltapException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/pipe/AbstractPipeFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.pipe; 6 | 7 | import com.airbnb.common.metrics.TaggedMetricRegistry; 8 | import com.airbnb.spinaltap.common.config.SourceConfiguration; 9 | import com.airbnb.spinaltap.common.source.SourceState; 10 | import com.airbnb.spinaltap.common.util.StateRepositoryFactory; 11 | import java.net.InetAddress; 12 | import java.net.UnknownHostException; 13 | import java.util.List; 14 | import lombok.RequiredArgsConstructor; 15 | import lombok.extern.slf4j.Slf4j; 16 | 17 | @Slf4j 18 | @RequiredArgsConstructor 19 | public abstract class AbstractPipeFactory { 20 | private static String HOST_NAME = "unknown"; 21 | 22 | protected final TaggedMetricRegistry metricRegistry; 23 | 24 | public abstract List createPipes( 25 | T sourceConfig, 26 | String partitionName, 27 | StateRepositoryFactory repositoryFactory, 28 | long leaderEpoch) 29 | throws Exception; 30 | 31 | protected static String getHostName() { 32 | if ("unknown".equalsIgnoreCase(HOST_NAME)) { 33 | try { 34 | HOST_NAME = InetAddress.getLocalHost().getCanonicalHostName(); 35 | } catch (UnknownHostException e) { 36 | log.error("Could not retrieve host name", e); 37 | } 38 | } 39 | 40 | return HOST_NAME; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/pipe/PipeMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.pipe; 6 | 7 | import com.airbnb.common.metrics.TaggedMetricRegistry; 8 | import com.airbnb.spinaltap.common.metrics.SpinalTapMetrics; 9 | import com.google.common.collect.ImmutableMap; 10 | 11 | /** Responsible for metrics collection for a {@link Pipe}. */ 12 | public class PipeMetrics extends SpinalTapMetrics { 13 | private static final String PIPE_PREFIX = METRIC_PREFIX + ".pipe"; 14 | 15 | private static final String OPEN_METRIC = PIPE_PREFIX + ".open.count"; 16 | private static final String CLOSE_METRIC = PIPE_PREFIX + ".close.count"; 17 | private static final String START_METRIC = PIPE_PREFIX + ".start.count"; 18 | private static final String STOP_METRIC = PIPE_PREFIX + ".stop.count"; 19 | private static final String CHECKPOINT_METRIC = PIPE_PREFIX + ".checkpoint.count"; 20 | 21 | public PipeMetrics(String sourceName, TaggedMetricRegistry metricRegistry) { 22 | this(ImmutableMap.of(SOURCE_NAME_TAG, sourceName), metricRegistry); 23 | } 24 | 25 | public PipeMetrics(ImmutableMap tags, TaggedMetricRegistry metricRegistry) { 26 | super(tags, metricRegistry); 27 | } 28 | 29 | public void open() { 30 | inc(OPEN_METRIC); 31 | } 32 | 33 | public void close() { 34 | inc(CLOSE_METRIC); 35 | } 36 | 37 | public void start() { 38 | inc(START_METRIC); 39 | } 40 | 41 | public void stop() { 42 | inc(STOP_METRIC); 43 | } 44 | 45 | public void checkpoint() { 46 | inc(CHECKPOINT_METRIC); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/AbstractDataStoreSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.util.ConcurrencyUtil; 9 | import com.airbnb.spinaltap.common.util.Filter; 10 | import com.airbnb.spinaltap.common.util.Mapper; 11 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 12 | import java.util.List; 13 | import java.util.concurrent.ExecutorService; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.TimeUnit; 16 | import javax.annotation.Nullable; 17 | import lombok.extern.slf4j.Slf4j; 18 | 19 | /** 20 | * Base implementation for Data Store source (Such as MySQL, DynamoDB). 21 | * 22 | * @param The event type produced by the source 23 | */ 24 | @Slf4j 25 | public abstract class AbstractDataStoreSource extends AbstractSource { 26 | private @Nullable ExecutorService processor; 27 | 28 | public AbstractDataStoreSource( 29 | String name, 30 | SourceMetrics metrics, 31 | Mapper>> mutationMapper, 32 | Filter eventFilter) { 33 | super(name, metrics, mutationMapper, eventFilter); 34 | } 35 | 36 | @Override 37 | protected synchronized void start() { 38 | processor = 39 | Executors.newSingleThreadExecutor( 40 | new ThreadFactoryBuilder().setNameFormat(name + "-source-processor").build()); 41 | 42 | processor.execute( 43 | () -> { 44 | try { 45 | connect(); 46 | } catch (Exception ex) { 47 | started.set(false); 48 | metrics.startFailure(ex); 49 | log.error("Failed to stream events for source " + name, ex); 50 | } 51 | }); 52 | } 53 | 54 | @Override 55 | protected void stop() throws Exception { 56 | if (isRunning()) { 57 | synchronized (this) { 58 | ConcurrencyUtil.shutdownGracefully(processor, 2, TimeUnit.SECONDS); 59 | } 60 | } 61 | disconnect(); 62 | } 63 | 64 | @Override 65 | public synchronized boolean isStarted() { 66 | return started.get() && isRunning(); 67 | } 68 | 69 | @Override 70 | protected synchronized boolean isRunning() { 71 | return processor != null && !processor.isShutdown(); 72 | } 73 | 74 | @Override 75 | protected synchronized boolean isTerminated() { 76 | return processor == null || processor.isTerminated(); 77 | } 78 | 79 | protected abstract void connect() throws Exception; 80 | 81 | protected abstract void disconnect() throws Exception; 82 | 83 | protected abstract boolean isConnected(); 84 | } 85 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/ListenableSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import lombok.NonNull; 11 | 12 | /** 13 | * Base {@link Source} implement using observer pattern to allow listening to 15 | * streamed events and subscribe to lifecycle change notifications. 16 | */ 17 | abstract class ListenableSource implements Source { 18 | private final List listeners = new ArrayList<>(); 19 | 20 | @Override 21 | public void addListener(@NonNull final Listener listener) { 22 | listeners.add(listener); 23 | } 24 | 25 | @Override 26 | public void removeListener(@NonNull final Listener listener) { 27 | listeners.remove(listener); 28 | } 29 | 30 | protected void notifyMutations(final List> mutations) { 31 | if (!mutations.isEmpty()) { 32 | listeners.forEach(listener -> listener.onMutation(mutations)); 33 | } 34 | } 35 | 36 | protected void notifyEvent(E event) { 37 | listeners.forEach(listener -> listener.onEvent(event)); 38 | } 39 | 40 | protected void notifyError(Throwable error) { 41 | listeners.forEach(listener -> listener.onError(error)); 42 | } 43 | 44 | protected void notifyStart() { 45 | listeners.forEach(Source.Listener::onStart); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/MysqlSourceState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 9 | import com.fasterxml.jackson.annotation.JsonProperty; 10 | import lombok.AllArgsConstructor; 11 | import lombok.Data; 12 | import lombok.EqualsAndHashCode; 13 | import lombok.NoArgsConstructor; 14 | 15 | /** 16 | * Represents the state of a {@link Source}, based on the last {@link SourceEvent} streamed. This is 17 | * used to mark the checkpoint for the {@link Source}, which will help indicate what position to 18 | * point to in the changelog on restart. 19 | * 20 | *

At the moment, the implement is coupled to binlog event state and therefore confined to {@code 21 | * MysqlSource} usage. 22 | */ 23 | @Data 24 | @EqualsAndHashCode(callSuper = true) 25 | @NoArgsConstructor 26 | @AllArgsConstructor 27 | @JsonIgnoreProperties(ignoreUnknown = true) 28 | public class MysqlSourceState extends SourceState { 29 | /** The timestamp of the last streamed {@link SourceEvent} in the changelog. */ 30 | @JsonProperty private long lastTimestamp; 31 | 32 | /** The offset of the last streamed {@link SourceEvent} in the changelog. */ 33 | @JsonProperty private long lastOffset; 34 | 35 | /** The {@link BinlogFilePos} of the last streamed {@link SourceEvent} in the changelog. */ 36 | @JsonProperty private BinlogFilePos lastPosition; 37 | 38 | public MysqlSourceState( 39 | final long lastTimestamp, 40 | final long lastOffset, 41 | final long currentLeaderEpoch, 42 | final BinlogFilePos lastPosition) { 43 | super(currentLeaderEpoch); 44 | this.lastTimestamp = lastTimestamp; 45 | this.lastOffset = lastOffset; 46 | this.lastPosition = lastPosition; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/Source.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.destination.Destination; 9 | import java.util.List; 10 | 11 | /** 12 | * Represents the originating end of the {@link com.airbnb.spinaltap.common.pipe.Pipe}, where {@link 13 | * SourceEvent}s are received and transformed to {@link Mutation}s. 14 | */ 15 | public interface Source { 16 | /** The name of the source */ 17 | String getName(); 18 | 19 | /** Adds a {@link Listener} to the source. */ 20 | void addListener(Listener listener); 21 | 22 | /** Removes a {@link Listener} from the source. */ 23 | void removeListener(Listener listener); 24 | 25 | /** Whether the source is started and processing events. */ 26 | boolean isStarted(); 27 | 28 | /** 29 | * Initializes the source and prepares for event processing. 30 | * 31 | *

The operation should be idempotent. 32 | */ 33 | void open(); 34 | 35 | /** 36 | * Stops event processing and closes the source. 37 | * 38 | *

The operation should be idempotent. 39 | */ 40 | void close(); 41 | 42 | /** 43 | * Clears the state of the source. 44 | * 45 | *

The operation should be idempotent. 46 | */ 47 | void clear(); 48 | 49 | /** 50 | * Commits the source checkpoint on the specified {@link Mutation}. On source start, streaming 51 | * will begin from the last marked checkpoint. 52 | */ 53 | void checkpoint(Mutation mutation); 54 | 55 | /** Represents a source listener to get notified of events and lifecycle changes. */ 56 | abstract class Listener { 57 | /** Action to perform after the {@link Destination} has started. */ 58 | public void onStart() {} 59 | 60 | /** Action to perform after a {@link SourceEvent}s has been received. */ 61 | public void onEvent(SourceEvent event) {} 62 | 63 | /** Action to perform after a {@link Mutation}s has been detected. */ 64 | public void onMutation(List> mutations) {} 65 | 66 | /** Action to perform when an error is caught on processing an event. */ 67 | public void onError(Throwable error) {} 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/SourceEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import lombok.AllArgsConstructor; 8 | import lombok.Getter; 9 | import lombok.NoArgsConstructor; 10 | import lombok.ToString; 11 | 12 | /** Represents a base event streamed from a {@link Source}. */ 13 | @Getter 14 | @ToString 15 | @NoArgsConstructor 16 | @AllArgsConstructor 17 | public abstract class SourceEvent { 18 | private long timestamp = System.currentTimeMillis(); 19 | 20 | /** Returns the number of entities in the event */ 21 | public int size() { 22 | return 1; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/source/SourceState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | import lombok.AllArgsConstructor; 10 | import lombok.Data; 11 | import lombok.NoArgsConstructor; 12 | 13 | @Data 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | @JsonIgnoreProperties(ignoreUnknown = true) 17 | public abstract class SourceState { 18 | /** 19 | * The leader epoch for the {@code Source}. The epoch acts as a high watermark, and is typically 20 | * incremented on leader election. 21 | * 22 | *

Note: This is only applicable if a cluster solution is employed. It is is used to mitigate 23 | * network partition (split brain) scenarios, and avoid having two cluster nodes concurrently 24 | * streaming from the same {@link Source}. 25 | */ 26 | @JsonProperty private long currentLeaderEpoch; 27 | } 28 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/BatchMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * Responsible for mapping a list of objects. 11 | * 12 | * @param The mapped from object type. 13 | * @param The mapped to object type. 14 | */ 15 | @FunctionalInterface 16 | public interface BatchMapper { 17 | /** 18 | * Applies the mapping function on the list of objects. 19 | * 20 | * @param objects the objects to map. 21 | * @return the mapped objects. 22 | */ 23 | List apply(List objects); 24 | } 25 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ChainedFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import lombok.AccessLevel; 10 | import lombok.NoArgsConstructor; 11 | import lombok.NonNull; 12 | import lombok.RequiredArgsConstructor; 13 | 14 | /** 15 | * Represents a chain of {@link Filter}s, where all {@link Filter} conditions need to pass. 16 | * 17 | * @param the filtered object type. 18 | */ 19 | @RequiredArgsConstructor 20 | public class ChainedFilter implements Filter { 21 | @NonNull private final List> filters; 22 | 23 | public static Builder builder() { 24 | return new Builder<>(); 25 | } 26 | 27 | /** 28 | * Applies the filters on the object. 29 | * 30 | * @param object the object to filter. 31 | * @return {@code true} if all filter conditions pass, {@code false} otherwise. 32 | */ 33 | public boolean apply(final T object) { 34 | return filters.stream().allMatch(filter -> filter.apply(object)); 35 | } 36 | 37 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 38 | public static final class Builder { 39 | private final List> filters = new ArrayList<>(); 40 | 41 | public Builder addFilter(Filter filter) { 42 | filters.add(filter); 43 | return this; 44 | } 45 | 46 | public Filter build() { 47 | return new ChainedFilter<>(filters); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ClassBasedMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import com.google.common.base.Preconditions; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | import lombok.AccessLevel; 11 | import lombok.NoArgsConstructor; 12 | import lombok.NonNull; 13 | import lombok.RequiredArgsConstructor; 14 | 15 | /** Maps an object according to the registered mapper by {@link Class} type. */ 16 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 17 | public class ClassBasedMapper implements Mapper { 18 | @NonNull private final Map, Mapper> locator; 19 | 20 | public static ClassBasedMapper.Builder builder() { 21 | return new ClassBasedMapper.Builder<>(); 22 | } 23 | 24 | @Override 25 | public R map(@NonNull final T object) { 26 | Mapper mapper = locator.get(object.getClass()); 27 | Preconditions.checkState(mapper != null, "No mapper found for type " + object.getClass()); 28 | 29 | return mapper.map(object); 30 | } 31 | 32 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 33 | public static class Builder { 34 | private final Map, Mapper> locator = 35 | new HashMap<>(); 36 | 37 | public Builder addMapper(Class klass, Mapper mapper) { 38 | locator.put(klass, mapper); 39 | return this; 40 | } 41 | 42 | @SuppressWarnings({"unchecked", "rawtypes"}) 43 | public Mapper build() { 44 | return new ClassBasedMapper(locator); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ConcurrencyUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.TimeUnit; 9 | import javax.validation.constraints.Min; 10 | import lombok.NonNull; 11 | import lombok.experimental.UtilityClass; 12 | 13 | /** Utility methods for concurrency operations */ 14 | @UtilityClass 15 | public class ConcurrencyUtil { 16 | /** 17 | * Attempts to shutdown the {@link ExecutorService}. If the service does not terminate within the 18 | * specified timeout, a force shutdown will be triggered. 19 | * 20 | * @param executorService the {@link ExecutorService}. 21 | * @param timeout the timeout. 22 | * @param unit the time unit. 23 | * @return {@code true} if shutdown was successful within the specified timeout, {@code false} 24 | * otherwise. 25 | */ 26 | public boolean shutdownGracefully( 27 | @NonNull ExecutorService executorService, @Min(1) long timeout, @NonNull TimeUnit unit) { 28 | boolean shutdown = false; 29 | executorService.shutdown(); 30 | try { 31 | shutdown = executorService.awaitTermination(timeout, unit); 32 | } catch (InterruptedException e) { 33 | executorService.shutdownNow(); 34 | } 35 | if (!shutdown) { 36 | executorService.shutdownNow(); 37 | } 38 | return shutdown; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ErrorHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import com.airbnb.spinaltap.common.exception.SpinaltapException; 8 | 9 | /** Responsible for handling {@code SpinaltapException}s. */ 10 | @FunctionalInterface 11 | public interface ErrorHandler { 12 | void handle(SpinaltapException e) throws SpinaltapException; 13 | } 14 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/Filter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | /** 8 | * Responsible for filtering object 9 | * 10 | * @param The filtered object type 11 | */ 12 | @FunctionalInterface 13 | public interface Filter { 14 | /** 15 | * Applies the filter on the object 16 | * 17 | * @param object the object to filter 18 | * @return {@code true} if the filter condition passes, {@code false} otherwise 19 | */ 20 | boolean apply(T object); 21 | } 22 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/Joiner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import javax.annotation.Nullable; 8 | 9 | /** 10 | * Joins two objects to produce a third. 11 | * 12 | * @param The first object type. 13 | * @param The second object type. 14 | * @param The result object type. 15 | */ 16 | @FunctionalInterface 17 | public interface Joiner { 18 | /** 19 | * Applies the joiner on a pair of objects 20 | * 21 | * @param first the first object to join 22 | * @param second the second object to join 23 | * @return the resulting joined object 24 | */ 25 | @Nullable 26 | R apply(@Nullable S first, @Nullable T second); 27 | } 28 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | import com.fasterxml.jackson.datatype.joda.JodaModule; 9 | import lombok.experimental.UtilityClass; 10 | 11 | /** Utility class for json operations and components. */ 12 | @UtilityClass 13 | public class JsonUtil { 14 | /** The {@link ObjectMapper} used for json SerDe. */ 15 | public ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 16 | 17 | static { 18 | OBJECT_MAPPER.registerModule(new JodaModule()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/KeyProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | /** 8 | * Responsible for providing a key for an object. 9 | * 10 | * @param The object type. 11 | * @param The key type. 12 | */ 13 | @FunctionalInterface 14 | public interface KeyProvider { 15 | /** 16 | * Gets the key for an object. 17 | * 18 | * @param object the object to get the key for. 19 | * @return the resulting key. 20 | */ 21 | R get(T object); 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/Mapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | /** 8 | * Responsible for mapping between objects. 9 | * 10 | * @param The mapped from object type. 11 | * @param The mapped to object type. 12 | */ 13 | @FunctionalInterface 14 | public interface Mapper { 15 | /** 16 | * Maps an object to another. 17 | * 18 | * @param object the object to map. 19 | * @return the resulting mapped object. 20 | */ 21 | R map(T object); 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/Repository.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | /** 8 | * Represents a single object data store repository. 9 | * 10 | * @param the object type stored in the repository. 11 | */ 12 | public interface Repository { 13 | /** @return whether an object exists */ 14 | boolean exists() throws Exception; 15 | 16 | /** 17 | * Creates a new object. 18 | * 19 | * @param value the object value. 20 | */ 21 | void create(T value) throws Exception; 22 | 23 | /** 24 | * Sets the current object value. 25 | * 26 | * @param value the object value. 27 | */ 28 | void set(T value) throws Exception; 29 | 30 | /** 31 | * Updates the current object value given a {@link DataUpdater}. 32 | * 33 | * @param value the object value. 34 | * @param updater the updater . 35 | */ 36 | void update(T value, DataUpdater updater) throws Exception; 37 | 38 | /** Retrieves the current object value. */ 39 | T get() throws Exception; 40 | 41 | /** Delete the current object */ 42 | void remove() throws Exception; 43 | 44 | /** 45 | * Responsible for determining the object value, as a function of the current value and new value. 46 | * 47 | * @param the object value type. 48 | */ 49 | interface DataUpdater { 50 | T apply(T currentValue, T newValue); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/StateRepositoryFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import com.airbnb.spinaltap.common.source.MysqlSourceState; 8 | import com.airbnb.spinaltap.common.source.SourceState; 9 | import java.util.Collection; 10 | 11 | /** Factory for {@link Repository}s that store {@link SourceState} objects. */ 12 | public interface StateRepositoryFactory { 13 | /** 14 | * @return the {@link Repository} of the {@link MysqlSourceState} object for a given resource and 15 | * partition. 16 | */ 17 | Repository getStateRepository(String resourceName, String partitionName); 18 | 19 | /** 20 | * @return the {@link Repository} of the history of {@link SourceState} objects for a given 21 | * resource and partition. 22 | */ 23 | Repository> getStateHistoryRepository(String resourceName, String partitionName); 24 | } 25 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ThreeWayJoiner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import javax.annotation.Nullable; 8 | 9 | /** 10 | * Joins three objects to produce a fourth. 11 | * 12 | * @param The first object type. 13 | * @param The second object type. 14 | * @param The third object type. 15 | * @param The result object type. 16 | */ 17 | @FunctionalInterface 18 | public interface ThreeWayJoiner { 19 | /** 20 | * Applies the joiner on a triplet of objects. 21 | * 22 | * @param first the first object to join. 23 | * @param second the second object to join. 24 | * @param third the third object to join. 25 | * @return the resulting joined object. 26 | */ 27 | @Nullable 28 | R apply(@Nullable X first, @Nullable Y second, @Nullable Z third); 29 | } 30 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/Validator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | /** 8 | * Responsible for validation logic on an object. 9 | * 10 | * @param The object type. 11 | */ 12 | public interface Validator { 13 | /** 14 | * Validates the object. 15 | * 16 | * @param object the object 17 | */ 18 | void validate(T object); 19 | 20 | /** Resets the state of the validator */ 21 | void reset(); 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/util/ZookeeperRepository.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import com.fasterxml.jackson.core.type.TypeReference; 8 | import lombok.NonNull; 9 | import lombok.RequiredArgsConstructor; 10 | import org.apache.curator.framework.CuratorFramework; 11 | 12 | /** 13 | * {@link Repository} implement with Zookeeper as backing store for objects. 14 | * 15 | * @param the object type. 16 | */ 17 | @RequiredArgsConstructor 18 | public class ZookeeperRepository implements Repository { 19 | @NonNull private final CuratorFramework zkClient; 20 | @NonNull private final String path; 21 | @NonNull private final TypeReference propertyClass; 22 | 23 | @Override 24 | public boolean exists() throws Exception { 25 | return zkClient.checkExists().forPath(path) != null; 26 | } 27 | 28 | @Override 29 | public void create(T data) throws Exception { 30 | zkClient 31 | .create() 32 | .creatingParentsIfNeeded() 33 | .forPath(path, JsonUtil.OBJECT_MAPPER.writeValueAsBytes(data)); 34 | } 35 | 36 | @Override 37 | public void set(T data) throws Exception { 38 | zkClient.setData().forPath(path, JsonUtil.OBJECT_MAPPER.writeValueAsBytes(data)); 39 | } 40 | 41 | @Override 42 | public void update(T data, DataUpdater updater) throws Exception { 43 | if (exists()) { 44 | set(updater.apply(get(), data)); 45 | } else { 46 | create(data); 47 | } 48 | } 49 | 50 | @Override 51 | public T get() throws Exception { 52 | byte[] value = zkClient.getData().forPath(path); 53 | return JsonUtil.OBJECT_MAPPER.readValue(value, propertyClass); 54 | } 55 | 56 | @Override 57 | public void remove() throws Exception { 58 | if (exists()) { 59 | zkClient.delete().guaranteed().forPath(path); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /spinaltap-common/src/main/java/com/airbnb/spinaltap/common/validator/MutationOrderValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.validator; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.util.Validator; 9 | import java.util.concurrent.atomic.AtomicLong; 10 | import java.util.function.Consumer; 11 | import lombok.NonNull; 12 | import lombok.RequiredArgsConstructor; 13 | import lombok.extern.slf4j.Slf4j; 14 | 15 | /** 16 | * Responsible for validating that {@link Mutation}s are streamed in order. This is typically used 17 | * as a {@link com.airbnb.spinaltap.common.source.Source.Listener} or {@link 18 | * com.airbnb.spinaltap.common.destination.Destination.Listener} to enforce {@link Mutation} 19 | * ordering guarantee in run-time. 20 | */ 21 | @Slf4j 22 | @RequiredArgsConstructor 23 | public final class MutationOrderValidator implements Validator> { 24 | /** The handler to execute on out-of-order {@link Mutation}. */ 25 | @NonNull private final Consumer> handler; 26 | 27 | /** The id of the last {@link Mutation} validated so far. */ 28 | private AtomicLong lastSeenId = new AtomicLong(-1); 29 | 30 | /** 31 | * Validates a {@link Mutation} is received in order, otherwise triggers the specified handler. 32 | */ 33 | @Override 34 | public void validate(@NonNull final Mutation mutation) { 35 | final long mutationId = mutation.getMetadata().getId(); 36 | log.debug("Validating order for mutation with id {}.", mutationId); 37 | 38 | if (lastSeenId.get() > mutationId) { 39 | log.warn( 40 | "Mutation with id {} is out of order and should precede {}.", mutationId, lastSeenId); 41 | handler.accept(mutation); 42 | } 43 | 44 | lastSeenId.set(mutationId); 45 | } 46 | 47 | /** Resets the state of the {@link Validator}. */ 48 | @Override 49 | public void reset() { 50 | lastSeenId.set(-1); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/destination/BufferedDestinationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import static org.junit.Assert.assertTrue; 8 | import static org.mockito.Mockito.mock; 9 | import static org.mockito.Mockito.verify; 10 | import static org.mockito.Mockito.when; 11 | 12 | import com.airbnb.spinaltap.Mutation; 13 | import com.google.common.collect.ImmutableList; 14 | import java.util.List; 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | 18 | public class BufferedDestinationTest { 19 | private final Mutation firstMutation = mock(Mutation.class); 20 | private final Mutation secondMutation = mock(Mutation.class); 21 | private final Mutation thirdMutation = mock(Mutation.class); 22 | 23 | private final List> mutations = 24 | ImmutableList.of(firstMutation, secondMutation, thirdMutation); 25 | 26 | private final Destination destination = mock(Destination.class); 27 | private final Destination.Listener listener = mock(Destination.Listener.class); 28 | private final DestinationMetrics metrics = mock(DestinationMetrics.class); 29 | 30 | private BufferedDestination bufferedDestination = 31 | new BufferedDestination("test", 10, destination, metrics); 32 | 33 | @Before 34 | public void setUp() throws Exception { 35 | bufferedDestination.addListener(listener); 36 | } 37 | 38 | @Test 39 | public void testOpenClose() throws Exception { 40 | when(destination.isStarted()).thenReturn(false); 41 | 42 | bufferedDestination.open(); 43 | 44 | when(destination.isStarted()).thenReturn(true); 45 | 46 | assertTrue(bufferedDestination.isStarted()); 47 | verify(destination).open(); 48 | 49 | bufferedDestination.open(); 50 | 51 | assertTrue(bufferedDestination.isStarted()); 52 | verify(destination).open(); 53 | 54 | bufferedDestination.close(); 55 | 56 | verify(destination).close(); 57 | } 58 | 59 | @Test 60 | public void testSend() throws Exception { 61 | when(destination.isStarted()).thenReturn(true); 62 | 63 | bufferedDestination.send(ImmutableList.of(firstMutation, secondMutation)); 64 | bufferedDestination.processMutations(); 65 | 66 | verify(destination).send(ImmutableList.of(firstMutation, secondMutation)); 67 | 68 | bufferedDestination.send(ImmutableList.of(firstMutation)); 69 | bufferedDestination.send(ImmutableList.of(secondMutation, thirdMutation)); 70 | bufferedDestination.processMutations(); 71 | 72 | verify(destination).send(mutations); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/destination/DestinationBuilderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | import static org.junit.Assert.assertTrue; 9 | import static org.mockito.Mockito.mock; 10 | 11 | import com.airbnb.common.metrics.TaggedMetricRegistry; 12 | import com.airbnb.spinaltap.Mutation; 13 | import com.airbnb.spinaltap.common.util.KeyProvider; 14 | import com.airbnb.spinaltap.common.util.Mapper; 15 | import lombok.NoArgsConstructor; 16 | import org.junit.Test; 17 | 18 | public class DestinationBuilderTest { 19 | private static final Mapper, Mutation> mapper = mutation -> mutation; 20 | private static final DestinationMetrics metrics = 21 | new DestinationMetrics("test", "test", new TaggedMetricRegistry()); 22 | 23 | @Test(expected = NullPointerException.class) 24 | public void testNoMapper() throws Exception { 25 | new TestDestinationBuilder().withMetrics(metrics).build(); 26 | } 27 | 28 | @Test(expected = NullPointerException.class) 29 | public void testNoMetrics() throws Exception { 30 | new TestDestinationBuilder().withMapper(mapper).build(); 31 | } 32 | 33 | @Test 34 | public void testBuildBufferedDestination() throws Exception { 35 | Destination destination = 36 | new TestDestinationBuilder().withMapper(mapper).withMetrics(metrics).withBuffer(5).build(); 37 | 38 | assertTrue(destination instanceof BufferedDestination); 39 | assertEquals(5, ((BufferedDestination) destination).getRemainingCapacity()); 40 | } 41 | 42 | @Test 43 | public void testBuildDestinationPool() throws Exception { 44 | Destination destination = 45 | new TestDestinationBuilder() 46 | .withMapper(mapper) 47 | .withMetrics(metrics) 48 | .withBuffer(5) 49 | .withPool(7, mock(KeyProvider.class)) 50 | .build(); 51 | 52 | assertTrue(destination instanceof DestinationPool); 53 | assertEquals(7, ((DestinationPool) destination).getPoolSize()); 54 | } 55 | 56 | @NoArgsConstructor 57 | class TestDestinationBuilder extends DestinationBuilder> { 58 | @Override 59 | protected Destination createDestination() { 60 | return mock(Destination.class); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/destination/ListenableDestinationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.destination; 6 | 7 | import static org.mockito.Mockito.mock; 8 | import static org.mockito.Mockito.verify; 9 | import static org.mockito.Mockito.verifyNoMoreInteractions; 10 | 11 | import com.airbnb.spinaltap.Mutation; 12 | import com.google.common.collect.ImmutableList; 13 | import java.util.List; 14 | import org.junit.Test; 15 | 16 | public class ListenableDestinationTest { 17 | private final Destination.Listener listener = mock(Destination.Listener.class); 18 | 19 | private ListenableDestination destination = new TestListenableDestination(); 20 | 21 | @Test 22 | public void test() throws Exception { 23 | Exception exception = mock(Exception.class); 24 | List> mutations = ImmutableList.of(mock(Mutation.class)); 25 | 26 | destination.addListener(listener); 27 | 28 | destination.notifyStart(); 29 | destination.notifySend(mutations); 30 | destination.notifyError(exception); 31 | 32 | verify(listener).onStart(); 33 | verify(listener).onSend(mutations); 34 | verify(listener).onError(exception); 35 | 36 | destination.removeListener(listener); 37 | 38 | destination.notifyStart(); 39 | destination.notifySend(mutations); 40 | destination.notifyError(exception); 41 | 42 | verifyNoMoreInteractions(listener); 43 | } 44 | 45 | private static final class TestListenableDestination extends ListenableDestination { 46 | @Override 47 | public Mutation getLastPublishedMutation() { 48 | return null; 49 | } 50 | 51 | @Override 52 | public void send(List> mutations) {} 53 | 54 | @Override 55 | public boolean isStarted() { 56 | return false; 57 | } 58 | 59 | @Override 60 | public void close() {} 61 | 62 | @Override 63 | public void clear() {} 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/pipe/PipeManagerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.pipe; 6 | 7 | import static org.junit.Assert.*; 8 | import static org.mockito.Mockito.*; 9 | 10 | import com.google.common.collect.ImmutableList; 11 | import org.junit.Test; 12 | 13 | public class PipeManagerTest { 14 | private static final String NAME = "test"; 15 | private static final String PARTITION = "test_0"; 16 | 17 | private final Pipe firstPipe = mock(Pipe.class); 18 | private final Pipe secondPipe = mock(Pipe.class); 19 | 20 | @Test 21 | public void testAddRemovePipe() throws Exception { 22 | PipeManager pipeManager = new PipeManager(); 23 | 24 | pipeManager.addPipes(NAME, PARTITION, ImmutableList.of(firstPipe, secondPipe)); 25 | 26 | verify(firstPipe, times(1)).start(); 27 | verify(secondPipe, times(1)).start(); 28 | 29 | pipeManager.removePipe(NAME, PARTITION); 30 | 31 | verify(firstPipe, times(1)).stop(); 32 | verify(secondPipe, times(1)).stop(); 33 | 34 | assertTrue(pipeManager.isEmpty()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/pipe/PipeTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.pipe; 6 | 7 | import static org.junit.Assert.*; 8 | import static org.mockito.Mockito.*; 9 | 10 | import com.airbnb.spinaltap.Mutation; 11 | import com.airbnb.spinaltap.common.destination.Destination; 12 | import com.airbnb.spinaltap.common.source.Source; 13 | import org.junit.Before; 14 | import org.junit.Test; 15 | 16 | public class PipeTest { 17 | private final Source source = mock(Source.class); 18 | private final Destination destination = mock(Destination.class); 19 | private final PipeMetrics metrics = mock(PipeMetrics.class); 20 | private final Mutation lastMutation = mock(Mutation.class); 21 | 22 | private final Pipe pipe = new Pipe(source, destination, metrics); 23 | 24 | @Before 25 | public void setUp() throws Exception { 26 | when(destination.getLastPublishedMutation()).thenReturn(lastMutation); 27 | } 28 | 29 | @Test 30 | public void testStartStop() throws Exception { 31 | Mutation mutation = mock(Mutation.class); 32 | Mutation.Metadata metadata = mock(Mutation.Metadata.class); 33 | 34 | when(destination.getLastPublishedMutation()).thenReturn(mutation); 35 | when(mutation.getMetadata()).thenReturn(metadata); 36 | 37 | pipe.start(); 38 | 39 | when(source.isStarted()).thenReturn(true); 40 | when(destination.isStarted()).thenReturn(true); 41 | 42 | verify(source, times(1)).addListener(any(Source.Listener.class)); 43 | verify(source, times(1)).open(); 44 | 45 | verify(destination, times(1)).addListener(any(Destination.Listener.class)); 46 | verify(destination, times(1)).open(); 47 | 48 | verify(metrics, times(1)).open(); 49 | 50 | pipe.stop(); 51 | 52 | verify(source, times(1)).removeListener(any(Source.Listener.class)); 53 | verify(source, times(1)).checkpoint(mutation); 54 | verify(source, times(1)).close(); 55 | 56 | verify(destination, times(1)).removeListener(any(Destination.Listener.class)); 57 | verify(destination, times(1)).close(); 58 | 59 | verify(metrics, times(1)).close(); 60 | } 61 | 62 | @Test 63 | public void testIsStarted() throws Exception { 64 | when(source.isStarted()).thenReturn(true); 65 | when(destination.isStarted()).thenReturn(false); 66 | 67 | assertFalse(pipe.isStarted()); 68 | 69 | when(source.isStarted()).thenReturn(false); 70 | when(destination.isStarted()).thenReturn(true); 71 | 72 | assertFalse(pipe.isStarted()); 73 | 74 | when(source.isStarted()).thenReturn(true); 75 | when(destination.isStarted()).thenReturn(true); 76 | 77 | assertTrue(pipe.isStarted()); 78 | } 79 | 80 | @Test 81 | public void testCheckpoint() throws Exception { 82 | pipe.checkpoint(); 83 | 84 | verify(source, times(1)).checkpoint(lastMutation); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/source/ListenableSourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.source; 6 | 7 | import static org.mockito.Mockito.mock; 8 | import static org.mockito.Mockito.verify; 9 | import static org.mockito.Mockito.verifyNoMoreInteractions; 10 | 11 | import com.airbnb.spinaltap.Mutation; 12 | import org.junit.Test; 13 | 14 | public class ListenableSourceTest { 15 | private final Source.Listener listener = mock(Source.Listener.class); 16 | 17 | private ListenableSource source = new TestListenableSource(); 18 | 19 | @Test 20 | public void test() throws Exception { 21 | Exception exception = mock(Exception.class); 22 | SourceEvent event = mock(SourceEvent.class); 23 | 24 | source.addListener(listener); 25 | 26 | source.notifyStart(); 27 | source.notifyEvent(event); 28 | source.notifyError(exception); 29 | 30 | verify(listener).onStart(); 31 | verify(listener).onEvent(event); 32 | verify(listener).onError(exception); 33 | 34 | source.removeListener(listener); 35 | 36 | source.notifyStart(); 37 | source.notifyEvent(event); 38 | source.notifyError(exception); 39 | 40 | verifyNoMoreInteractions(listener); 41 | } 42 | 43 | private static final class TestListenableSource extends ListenableSource { 44 | @Override 45 | public String getName() { 46 | return null; 47 | } 48 | 49 | @Override 50 | public boolean isStarted() { 51 | return false; 52 | } 53 | 54 | @Override 55 | public void open() {} 56 | 57 | @Override 58 | public void close() {} 59 | 60 | @Override 61 | public void clear() {} 62 | 63 | @Override 64 | public void checkpoint(Mutation mutation) {} 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/util/ChainedFilterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import static org.junit.Assert.assertFalse; 8 | import static org.junit.Assert.assertTrue; 9 | 10 | import org.junit.Test; 11 | 12 | public class ChainedFilterTest { 13 | @Test 14 | public void testFailingFilter() throws Exception { 15 | Filter filter = 16 | ChainedFilter.builder().addFilter(num -> true).addFilter(num -> false).build(); 17 | 18 | assertFalse(filter.apply(1)); 19 | } 20 | 21 | @Test 22 | public void testPassingFilter() throws Exception { 23 | Filter filter = 24 | ChainedFilter.builder().addFilter(num -> true).addFilter(num -> true).build(); 25 | 26 | assertTrue(filter.apply(1)); 27 | } 28 | 29 | @Test 30 | public void testEmptyFilter() throws Exception { 31 | Filter filter = ChainedFilter.builder().build(); 32 | 33 | assertTrue(filter.apply(1)); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/util/ClassBasedMapperTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.util; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import org.junit.Test; 10 | 11 | public class ClassBasedMapperTest { 12 | @Test 13 | public void testMap() throws Exception { 14 | Mapper mapper = 15 | ClassBasedMapper.builder() 16 | .addMapper(Float.class, Math::round) 17 | .addMapper(String.class, Integer::parseInt) 18 | .build(); 19 | 20 | assertEquals(new Integer(1), mapper.map(new Float(1.2))); 21 | assertEquals(new Integer(3), mapper.map("3")); 22 | } 23 | 24 | @Test(expected = IllegalStateException.class) 25 | public void testNoMapFound() throws Exception { 26 | ClassBasedMapper.builder().build().map(1); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /spinaltap-common/src/test/java/com/airbnb/spinaltap/common/validator/MutationOrderValidatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.common.validator; 6 | 7 | import static org.junit.Assert.*; 8 | import static org.mockito.Mockito.*; 9 | 10 | import com.airbnb.spinaltap.Mutation; 11 | import com.google.common.collect.Lists; 12 | import java.util.Arrays; 13 | import java.util.List; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | 17 | public class MutationOrderValidatorTest { 18 | private final Mutation firstMutation = mock(Mutation.class); 19 | private final Mutation secondMutation = mock(Mutation.class); 20 | 21 | private final Mutation.Metadata firstMetadata = mock(Mutation.Metadata.class); 22 | private final Mutation.Metadata secondMetadata = mock(Mutation.Metadata.class); 23 | 24 | @Before 25 | public void setUp() throws Exception { 26 | when(firstMutation.getMetadata()).thenReturn(firstMetadata); 27 | when(secondMutation.getMetadata()).thenReturn(secondMetadata); 28 | } 29 | 30 | @Test 31 | public void testMutationInOrder() throws Exception { 32 | List unorderedMutations = Lists.newArrayList(); 33 | 34 | when(firstMetadata.getId()).thenReturn(1L); 35 | when(secondMetadata.getId()).thenReturn(2L); 36 | 37 | MutationOrderValidator validator = new MutationOrderValidator(unorderedMutations::add); 38 | 39 | validator.validate(firstMutation); 40 | validator.validate(secondMutation); 41 | 42 | assertTrue(unorderedMutations.isEmpty()); 43 | } 44 | 45 | @Test 46 | public void testMutationOutOfOrder() throws Exception { 47 | List unorderedMutations = Lists.newArrayList(); 48 | 49 | when(firstMetadata.getId()).thenReturn(2L); 50 | when(secondMetadata.getId()).thenReturn(1L); 51 | 52 | MutationOrderValidator validator = new MutationOrderValidator(unorderedMutations::add); 53 | 54 | validator.validate(firstMutation); 55 | validator.validate(secondMutation); 56 | 57 | assertEquals(Arrays.asList(secondMutation), unorderedMutations); 58 | } 59 | 60 | @Test 61 | public void testReset() throws Exception { 62 | List unorderedMutations = Lists.newArrayList(); 63 | 64 | when(firstMetadata.getId()).thenReturn(1L); 65 | when(secondMetadata.getId()).thenReturn(2L); 66 | 67 | MutationOrderValidator validator = new MutationOrderValidator(unorderedMutations::add); 68 | 69 | validator.validate(firstMutation); 70 | validator.validate(secondMutation); 71 | 72 | validator.reset(); 73 | 74 | validator.validate(firstMutation); 75 | validator.validate(secondMutation); 76 | 77 | assertTrue(unorderedMutations.isEmpty()); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /spinaltap-kafka/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile project(":spinaltap-common") 3 | compile project(":spinaltap-mysql") 4 | compile project(":kafka-test-harness") 5 | compile libraries.kafka_core 6 | compileOnly libraries.lombok 7 | annotationProcessor libraries.lombok 8 | 9 | testCompile libraries.junit 10 | } -------------------------------------------------------------------------------- /spinaltap-kafka/src/main/java/com/airbnb/spinaltap/kafka/KafkaDestinationBuilder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.kafka; 6 | 7 | import com.airbnb.spinaltap.common.destination.Destination; 8 | import com.airbnb.spinaltap.common.destination.DestinationBuilder; 9 | import lombok.NonNull; 10 | import lombok.RequiredArgsConstructor; 11 | import org.apache.thrift.TBase; 12 | 13 | /** Represents an implement of {@link DestinationBuilder} for {@link KafkaDestination}s. */ 14 | @RequiredArgsConstructor 15 | public final class KafkaDestinationBuilder> extends DestinationBuilder { 16 | @NonNull private final KafkaProducerConfiguration producerConfig; 17 | 18 | @Override 19 | protected Destination createDestination() { 20 | return new KafkaDestination<>(topicNamePrefix, producerConfig, mapper, metrics, delaySendMs); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-kafka/src/main/java/com/airbnb/spinaltap/kafka/KafkaProducerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.kafka; 6 | 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import lombok.AllArgsConstructor; 9 | import lombok.Data; 10 | import lombok.NoArgsConstructor; 11 | 12 | /** Represents the Kafka producer configuration used in {@link KafkaDestination}. */ 13 | @Data 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | public class KafkaProducerConfiguration { 17 | @JsonProperty("bootstrap_servers") 18 | private String bootstrapServers; 19 | } 20 | -------------------------------------------------------------------------------- /spinaltap-model/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "org.jruyi.thrift" version "0.4.0" 3 | } 4 | 5 | dependencies { 6 | compile libraries.apache_commons_lang 7 | compile libraries.guava 8 | compile libraries.jackson_annotations 9 | compile libraries.jackson_databind 10 | compile libraries.slf4j 11 | compile libraries.thrift 12 | compileOnly libraries.lombok 13 | annotationProcessor libraries.lombok 14 | 15 | testCompile libraries.junit 16 | } 17 | 18 | compileThrift { 19 | createGenFolder false 20 | generator "java", "beans,fullcamel" 21 | } 22 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/Mutation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import com.google.common.collect.ImmutableSet; 8 | import com.google.common.collect.Sets; 9 | import java.util.Map; 10 | import java.util.Objects; 11 | import java.util.Set; 12 | import java.util.stream.Collectors; 13 | import lombok.Getter; 14 | import lombok.RequiredArgsConstructor; 15 | import lombok.ToString; 16 | 17 | /** 18 | * Base class which represents a data entity change (mutation). 19 | * 20 | * @param The data entity type (ex: Row, Record, etc...) 21 | */ 22 | @Getter 23 | @ToString 24 | @RequiredArgsConstructor 25 | public abstract class Mutation { 26 | private static final byte INSERT_BYTE = 0x1; 27 | private static final byte UPDATE_BYTE = 0x2; 28 | private static final byte DELETE_BYTE = 0x3; 29 | private static final byte INVALID_BYTE = 0x4; 30 | 31 | @Getter 32 | @RequiredArgsConstructor 33 | public enum Type { 34 | INSERT(INSERT_BYTE), 35 | UPDATE(UPDATE_BYTE), 36 | DELETE(DELETE_BYTE), 37 | INVALID(INVALID_BYTE); 38 | 39 | final byte code; 40 | 41 | public static Type fromCode(byte code) { 42 | switch (code) { 43 | case INSERT_BYTE: 44 | return INSERT; 45 | case UPDATE_BYTE: 46 | return UPDATE; 47 | case DELETE_BYTE: 48 | return DELETE; 49 | default: 50 | return INVALID; 51 | } 52 | } 53 | } 54 | 55 | private final Metadata metadata; 56 | private final Type type; 57 | private final T entity; 58 | 59 | @Getter 60 | @ToString 61 | @RequiredArgsConstructor 62 | public abstract static class Metadata { 63 | private final long id; 64 | private final long timestamp; 65 | } 66 | 67 | // For use by subclasses that implement a mutation with type UPDATE. 68 | protected static Set getUpdatedColumns( 69 | final Map previousValues, final Map currentValues) { 70 | final Set previousColumns = previousValues.keySet(); 71 | final Set currentColumns = currentValues.keySet(); 72 | 73 | return ImmutableSet.builder() 74 | .addAll(Sets.symmetricDifference(currentColumns, previousColumns)) 75 | .addAll( 76 | Sets.intersection(currentColumns, previousColumns) 77 | .stream() 78 | .filter( 79 | column -> 80 | // Use deepEquals to allow testing for equality between two byte arrays. 81 | !Objects.deepEquals(previousValues.get(column), currentValues.get(column))) 82 | .collect(Collectors.toSet())) 83 | .build(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/DataSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnore; 8 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 9 | import lombok.AllArgsConstructor; 10 | import lombok.EqualsAndHashCode; 11 | import lombok.Getter; 12 | import lombok.NoArgsConstructor; 13 | import lombok.ToString; 14 | 15 | /** Represents a MySQL data source configuration. */ 16 | @Getter 17 | @ToString 18 | @NoArgsConstructor 19 | @EqualsAndHashCode 20 | @AllArgsConstructor 21 | @JsonIgnoreProperties(ignoreUnknown = true) 22 | public class DataSource { 23 | private String host; 24 | private int port; 25 | private String service; 26 | 27 | @JsonIgnore 28 | @Getter(lazy = true) 29 | private final com.airbnb.jitney.event.spinaltap.v1.DataSource thriftDataSource = 30 | toThriftDataSource(this); 31 | 32 | private static com.airbnb.jitney.event.spinaltap.v1.DataSource toThriftDataSource( 33 | DataSource dataSource) { 34 | return new com.airbnb.jitney.event.spinaltap.v1.DataSource( 35 | dataSource.getHost(), dataSource.getPort(), dataSource.getService()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/Transaction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.Value; 9 | 10 | /** Represents a MySQL Transaction boundary in the binlog. */ 11 | @Value 12 | @RequiredArgsConstructor 13 | public class Transaction { 14 | private final long timestamp; 15 | private final long offset; 16 | private final BinlogFilePos position; 17 | private final String gtid; 18 | 19 | public Transaction(long timestamp, long offset, BinlogFilePos position) { 20 | this.timestamp = timestamp; 21 | this.offset = offset; 22 | this.position = position; 23 | this.gtid = null; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlDeleteMutation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 9 | import java.util.Set; 10 | 11 | public final class MysqlDeleteMutation extends MysqlMutation { 12 | public MysqlDeleteMutation(MysqlMutationMetadata metadata, Row row) { 13 | super(metadata, Mutation.Type.DELETE, row); 14 | } 15 | 16 | @Override 17 | public Set getChangedColumns() { 18 | return getRow().getColumns().keySet(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlInsertMutation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 8 | import java.util.Set; 9 | 10 | public final class MysqlInsertMutation extends MysqlMutation { 11 | public MysqlInsertMutation(MysqlMutationMetadata metadata, Row row) { 12 | super(metadata, Type.INSERT, row); 13 | } 14 | 15 | @Override 16 | public Set getChangedColumns() { 17 | return getRow().getColumns().keySet(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlMutation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 9 | import java.util.Set; 10 | import lombok.ToString; 11 | 12 | /** Represents a MySQL {@link Mutation} derived from a binlog event. */ 13 | @ToString(callSuper = true) 14 | public abstract class MysqlMutation extends Mutation { 15 | public MysqlMutation(MysqlMutationMetadata metadata, Mutation.Type type, Row row) { 16 | super(metadata, type, row); 17 | } 18 | 19 | public final Row getRow() { 20 | return getEntity(); 21 | } 22 | 23 | /** @return columns of the table that have changed value as a result of this mutation */ 24 | public abstract Set getChangedColumns(); 25 | 26 | @Override 27 | public final MysqlMutationMetadata getMetadata() { 28 | return (MysqlMutationMetadata) super.getMetadata(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlMutationMetadata.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 9 | import com.airbnb.spinaltap.mysql.DataSource; 10 | import com.airbnb.spinaltap.mysql.Transaction; 11 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 12 | import lombok.EqualsAndHashCode; 13 | import lombok.ToString; 14 | import lombok.Value; 15 | 16 | @Value 17 | @ToString(callSuper = true) 18 | @EqualsAndHashCode(callSuper = true) 19 | public class MysqlMutationMetadata extends Mutation.Metadata { 20 | private final DataSource dataSource; 21 | private final BinlogFilePos filePos; 22 | private final Table table; 23 | private final long serverId; 24 | private final Transaction beginTransaction; 25 | private final Transaction lastTransaction; 26 | 27 | /** The leader epoch of the node resource processing the event. */ 28 | private final long leaderEpoch; 29 | 30 | /** The mutation row position in the given binlog event. */ 31 | private final int eventRowPosition; 32 | 33 | public MysqlMutationMetadata( 34 | DataSource dataSource, 35 | BinlogFilePos filePos, 36 | Table table, 37 | long serverId, 38 | long id, 39 | long timestamp, 40 | Transaction beginTransaction, 41 | Transaction lastTransaction, 42 | long leaderEpoch, 43 | int eventRowPosition) { 44 | super(id, timestamp); 45 | 46 | this.dataSource = dataSource; 47 | this.filePos = filePos; 48 | this.table = table; 49 | this.serverId = serverId; 50 | this.beginTransaction = beginTransaction; 51 | this.lastTransaction = lastTransaction; 52 | this.leaderEpoch = leaderEpoch; 53 | this.eventRowPosition = eventRowPosition; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlUpdateMutation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 9 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 10 | import com.google.common.annotations.VisibleForTesting; 11 | import com.google.common.collect.Maps; 12 | import java.io.Serializable; 13 | import java.util.Map; 14 | import java.util.Set; 15 | import lombok.Getter; 16 | import lombok.NonNull; 17 | import lombok.ToString; 18 | 19 | @Getter 20 | @ToString(callSuper = true) 21 | public final class MysqlUpdateMutation extends MysqlMutation { 22 | private final Row previousRow; 23 | 24 | public MysqlUpdateMutation( 25 | final MysqlMutationMetadata metadata, final Row previousRow, final Row row) { 26 | super(metadata, Mutation.Type.UPDATE, row); 27 | 28 | this.previousRow = previousRow; 29 | } 30 | 31 | @Override 32 | public Set getChangedColumns() { 33 | // Transform the column values of each Row to Map. Map values of type 34 | // byte[], or columns of type BLOB, will be tested for equality using deepEquals in method 35 | // getUpdatedColumns. If we simply passed down the Map of each Row, then 36 | // deepEquals would in turn call the equals method of type Column, which will wrongly not use 37 | // deepEquals to compare byte[] values. 38 | return Mutation.getUpdatedColumns(asColumnValues(getPreviousRow()), asColumnValues(getRow())); 39 | } 40 | 41 | @VisibleForTesting 42 | static Map asColumnValues(@NonNull final Row row) { 43 | return Maps.transformValues(row.getColumns(), Column::getValue); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/schema/Column.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.schema; 6 | 7 | import java.io.Serializable; 8 | import lombok.Value; 9 | 10 | /** Represents a MySQL column. */ 11 | @Value 12 | public class Column { 13 | private final ColumnMetadata metadata; 14 | private final Serializable value; 15 | } 16 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/schema/ColumnDataType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.schema; 6 | 7 | import java.util.Map; 8 | import java.util.function.Function; 9 | import java.util.stream.Collectors; 10 | import java.util.stream.Stream; 11 | import lombok.Getter; 12 | import lombok.RequiredArgsConstructor; 13 | 14 | /** An enumeration of the supported data types for a MySQL {@link Column}. */ 15 | @RequiredArgsConstructor 16 | public enum ColumnDataType { 17 | DECIMAL(0), 18 | TINY(1), 19 | SHORT(2), 20 | LONG(3), 21 | FLOAT(4), 22 | DOUBLE(5), 23 | NULL(6), 24 | TIMESTAMP(7), 25 | LONGLONG(8), 26 | INT24(9), 27 | DATE(10), 28 | TIME(11), 29 | DATETIME(12), 30 | YEAR(13), 31 | NEWDATE(14), 32 | VARCHAR(15), 33 | BIT(16), 34 | // (TIMESTAMP|DATETIME|TIME)_V2 data types appeared in MySQL 5.6.4 35 | // @see http://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html 36 | TIMESTAMP_V2(17), 37 | DATETIME_V2(18), 38 | TIME_V2(19), 39 | NEWDECIMAL(246), 40 | ENUM(247), 41 | SET(248), 42 | TINY_BLOB(249), 43 | MEDIUM_BLOB(250), 44 | LONG_BLOB(251), 45 | BLOB(252), 46 | VAR_STRING(253), 47 | STRING(254), 48 | GEOMETRY(255), 49 | // TODO: Remove UNKNOWN. This is known finite list of types. Any other value is not valid. 50 | // Treating this case as an error (throwing exception etc) is better. 51 | UNKNOWN(-1); 52 | 53 | @Getter private final int code; 54 | 55 | private static final Map INDEX_BY_CODE = 56 | Stream.of(values()).collect(Collectors.toMap(ColumnDataType::getCode, Function.identity())); 57 | 58 | /** 59 | * The Java type is an 8-bit signed two's complement integer. But MySql byte is not. So when 60 | * looking up the column type from a MySql byte, it must be upcast to an int first. 61 | * 62 | *

As an example, let's take BLOB/252. Mysql column type will be the byte 0b11111100. If casted 63 | * to a java integer it will be interpreted as -4. 64 | * 65 | *

Integer.toBinaryString((int)((byte) 252)): '11111111111111111111111111111100' 66 | */ 67 | public static ColumnDataType byCode(final byte code) { 68 | return byCode(code & 0xFF); 69 | } 70 | 71 | public static ColumnDataType byCode(final int code) { 72 | return INDEX_BY_CODE.getOrDefault(code, UNKNOWN); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/schema/ColumnMetadata.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.schema; 6 | 7 | import lombok.AllArgsConstructor; 8 | import lombok.Data; 9 | import lombok.RequiredArgsConstructor; 10 | 11 | /** Represents additional metadata on a MySQL {@link Column}. */ 12 | @Data 13 | @RequiredArgsConstructor 14 | @AllArgsConstructor 15 | public class ColumnMetadata { 16 | private final String name; 17 | private final ColumnDataType colType; 18 | private final boolean isPrimaryKey; 19 | private final int position; 20 | private String rawColumnType; 21 | } 22 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/schema/PrimaryKey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.schema; 6 | 7 | import com.google.common.collect.ImmutableMap; 8 | import lombok.RequiredArgsConstructor; 9 | import lombok.Value; 10 | 11 | @Value 12 | @RequiredArgsConstructor 13 | public class PrimaryKey { 14 | /** Note: Insertion order should be preserved in the choice of {@link java.util.Map} implement. */ 15 | private final ImmutableMap columns; 16 | } 17 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/java/com/airbnb/spinaltap/mysql/mutation/schema/Row.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.schema; 6 | 7 | import com.google.common.collect.ImmutableMap; 8 | import lombok.Value; 9 | 10 | /** Represents a MySQL row. */ 11 | @Value 12 | public final class Row { 13 | private final Table table; 14 | private final ImmutableMap columns; 15 | 16 | @SuppressWarnings("unchecked") 17 | public T getValue(final String columnName) { 18 | return (T) columns.get(columnName).getValue(); 19 | } 20 | 21 | public String getPrimaryKeyValue() { 22 | if (!table.getPrimaryKey().isPresent()) { 23 | return null; 24 | } 25 | 26 | final StringBuilder value = new StringBuilder(); 27 | table 28 | .getPrimaryKey() 29 | .get() 30 | .getColumns() 31 | .keySet() 32 | .stream() 33 | .map(columns::get) 34 | .map(Column::getValue) 35 | .forEach(value::append); 36 | 37 | return value.toString(); 38 | } 39 | 40 | public boolean containsColumn(final String columnName) { 41 | return columns.containsKey(columnName); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spinaltap-model/src/main/thrift/com/airbnb/jitney/event/spinaltap/spinaltap_v1.thrift: -------------------------------------------------------------------------------- 1 | namespace java com.airbnb.jitney.event.spinaltap.v1 2 | namespace rb airbnb.jitney.event.spinaltap.v1 3 | 4 | enum MutationType { 5 | INSERT = 0x1 6 | UPDATE 7 | DELETE 8 | INVALID 9 | } 10 | 11 | struct DataSource { 12 | 1: required string hostname, 13 | 2: required i32 port, 14 | 3: required string synapse_service, 15 | } 16 | 17 | struct Column { 18 | 1: required i64 type, 19 | 2: required bool is_primary_key = false, 20 | 3: required string name, 21 | 4: optional i32 position, 22 | } 23 | 24 | struct BinlogHeader { 25 | 1: required string pos, 26 | 2: required i64 server_id, 27 | 3: required i64 timestamp, 28 | 4: required i32 type, 29 | 5: optional string last_transaction_pos, 30 | 6: optional i64 last_transaction_timestamp, 31 | 7: optional i64 leader_epoch, 32 | 8: optional i64 id, 33 | 9: optional i32 event_row_position, 34 | 10: optional string server_uuid, 35 | 11: optional string last_transaction_gtid_set, 36 | 12: optional string begin_transaction_pos, 37 | 13: optional i64 begin_transaction_timestamp, 38 | 14: optional string begin_transaction_gtid, 39 | } 40 | 41 | struct Table { 42 | 1: required i64 id, 43 | 2: required string name, 44 | 3: required string database, 45 | 4: required set primary_key, 46 | 5: required map columns, 47 | // When `overriding_database` is set, mutations will be published to the Kafka topic 48 | // with name .. instead of 49 | // .. 50 | 6: optional string overriding_database, 51 | } 52 | 53 | struct Mutation { 54 | 31337: optional string schema = "com.airbnb.jitney.event.spinaltap:Mutation:1.0.0", 55 | 1: required MutationType type, 56 | 2: required i64 timestamp, 57 | 3: required string source_id, 58 | 4: required DataSource data_source, 59 | 5: required BinlogHeader binlog_header, 60 | 6: required Table table, 61 | 7: required map entity, 62 | 8: optional map previous_entity, 63 | } 64 | -------------------------------------------------------------------------------- /spinaltap-model/src/test/java/com/airbnb/spinaltap/GtidSetTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | import static org.junit.Assert.assertNotEquals; 9 | 10 | import com.airbnb.spinaltap.mysql.GtidSet; 11 | import org.junit.Test; 12 | 13 | public class GtidSetTest { 14 | private static final String SERVER_UUID_1 = "665ef2f4-b008-4440-b78c-26ba7ce500e6"; 15 | private static final String SERVER_UUID_2 = "eeb24231-ff9d-4051-b9b1-bf40bf33b2be"; 16 | 17 | @Test 18 | public void testEmptySet() { 19 | assertEquals(new GtidSet("").toString(), ""); 20 | } 21 | 22 | @Test 23 | public void testEquals() { 24 | assertEquals(new GtidSet(""), new GtidSet("")); 25 | assertEquals(new GtidSet(""), new GtidSet(null)); 26 | 27 | assertEquals(new GtidSet(SERVER_UUID_1 + ":1-888"), new GtidSet(SERVER_UUID_1 + ":1-888")); 28 | 29 | GtidSet gtidSet1 = 30 | new GtidSet(String.format("%s:1-1023,%s:1-888", SERVER_UUID_1, SERVER_UUID_2)); 31 | GtidSet gtidSet2 = 32 | new GtidSet(String.format("%s:1-888,%s:1-1023", SERVER_UUID_2, SERVER_UUID_1)); 33 | assertEquals(gtidSet1, gtidSet2); 34 | assertEquals(gtidSet1.toString(), gtidSet2.toString()); 35 | 36 | assertNotEquals( 37 | new GtidSet(SERVER_UUID_1 + ":1-888"), new GtidSet(SERVER_UUID_1 + ":1-100:102-888")); 38 | assertNotEquals(new GtidSet(SERVER_UUID_1 + ":1-888"), new GtidSet(SERVER_UUID_2 + ":1-888")); 39 | } 40 | 41 | @Test 42 | public void testCollapseIntervals() { 43 | GtidSet gtidSet = new GtidSet(SERVER_UUID_1 + ":1-123:124:125-200"); 44 | assertEquals(gtidSet, new GtidSet(SERVER_UUID_1 + ":1-200")); 45 | assertEquals(gtidSet.toString(), SERVER_UUID_1 + ":1-200"); 46 | 47 | gtidSet = new GtidSet(SERVER_UUID_1 + ":1-201:202-211:239-244:245-300:400-409"); 48 | assertEquals(gtidSet, new GtidSet(SERVER_UUID_1 + ":1-211:239-300:400-409")); 49 | assertEquals(gtidSet.toString(), SERVER_UUID_1 + ":1-211:239-300:400-409"); 50 | 51 | gtidSet = new GtidSet(SERVER_UUID_1 + ":1-200:100-123:40-255:40-100:60-100:280-290:270-279"); 52 | assertEquals(gtidSet.toString(), SERVER_UUID_1 + ":1-255:270-290"); 53 | } 54 | 55 | @Test 56 | public void testMixedCaseServerUUID() { 57 | String upperCaseServerUUID1 = SERVER_UUID_1.toUpperCase(); 58 | GtidSet gtidSet = 59 | new GtidSet( 60 | String.format( 61 | "%s:1-24,%s:25-706,%s:1-23", upperCaseServerUUID1, SERVER_UUID_1, SERVER_UUID_2)); 62 | assertEquals( 63 | new GtidSet(String.format("%s:1-706,%s:1-23", SERVER_UUID_1, SERVER_UUID_2)), gtidSet); 64 | } 65 | 66 | @Test 67 | public void testSubsetOf() { 68 | GtidSet[] set = { 69 | new GtidSet(""), 70 | new GtidSet(SERVER_UUID_1 + ":1-191"), 71 | new GtidSet(SERVER_UUID_1 + ":192-199"), 72 | new GtidSet(SERVER_UUID_1 + ":1-191:192-199"), 73 | new GtidSet(SERVER_UUID_1 + ":1-191:193-199"), 74 | new GtidSet(SERVER_UUID_1 + ":2-199"), 75 | new GtidSet(SERVER_UUID_1 + ":1-200") 76 | }; 77 | byte[][] subsetMatrix = { 78 | {1, 1, 1, 1, 1, 1, 1}, 79 | {0, 1, 0, 1, 1, 0, 1}, 80 | {0, 0, 1, 1, 0, 1, 1}, 81 | {0, 0, 0, 1, 0, 0, 1}, 82 | {0, 0, 0, 1, 1, 0, 1}, 83 | {0, 0, 0, 1, 0, 1, 1}, 84 | {0, 0, 0, 0, 0, 0, 1}, 85 | }; 86 | for (int i = 0; i < subsetMatrix.length; i++) { 87 | byte[] subset = subsetMatrix[i]; 88 | for (int j = 0; j < subset.length; j++) { 89 | assertEquals(set[i].isContainedWithin(set[j]), subset[j] == 1); 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /spinaltap-model/src/test/java/com/airbnb/spinaltap/MutationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import com.google.common.collect.ImmutableMap; 10 | import com.google.common.collect.ImmutableSet; 11 | import java.util.Map; 12 | import org.junit.Test; 13 | 14 | public class MutationTest { 15 | private static final String SHARED_KEY = "sharedKey"; 16 | 17 | @Test 18 | public void testAddAndRemoveScalarColumns() { 19 | String removedKey = "removedKey"; 20 | String addedKey = "addedKey"; 21 | Map previousColumns = 22 | ImmutableMap.of( 23 | removedKey, "removedValue", 24 | SHARED_KEY, "sharedValue"); 25 | Map currentColumns = 26 | ImmutableMap.of( 27 | SHARED_KEY, "sharedValue", 28 | addedKey, "addedValue"); 29 | assertEquals( 30 | ImmutableSet.of(removedKey, addedKey), 31 | Mutation.getUpdatedColumns(previousColumns, currentColumns)); 32 | } 33 | 34 | @Test 35 | public void testUpdateScalarColumnValues() { 36 | String updatedKey = "updatedKey"; 37 | Map previousColumns = 38 | ImmutableMap.of( 39 | SHARED_KEY, "sharedValue", 40 | updatedKey, "previousValue"); 41 | Map currentColumns = 42 | ImmutableMap.of( 43 | SHARED_KEY, "sharedValue", 44 | updatedKey, "currentValue"); 45 | assertEquals( 46 | ImmutableSet.of(updatedKey), Mutation.getUpdatedColumns(previousColumns, currentColumns)); 47 | } 48 | 49 | @Test 50 | public void testUpdateArrayColumnValues() { 51 | String updatedKey = "updatedKey"; 52 | Map previousColumns = 53 | ImmutableMap.of( 54 | SHARED_KEY, new byte[] {0x00, 0x01}, 55 | updatedKey, new byte[] {0x02, 0x03}); 56 | Map currentColumns = 57 | ImmutableMap.of( 58 | SHARED_KEY, new byte[] {0x00, 0x01}, 59 | updatedKey, new byte[] {0x04, 0x05}); 60 | assertEquals( 61 | ImmutableSet.of(updatedKey), Mutation.getUpdatedColumns(previousColumns, currentColumns)); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /spinaltap-model/src/test/java/com/airbnb/spinaltap/mysql/BinlogFilePosTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import static org.junit.Assert.*; 8 | 9 | import org.junit.Test; 10 | 11 | public class BinlogFilePosTest { 12 | private static final String UUID1 = "07592619-e257-4033-a30f-7fe9fcfbf229"; 13 | private static final String UUID2 = "4a4ac150-fe5b-4093-a1ef-a8876011adaa"; 14 | 15 | @Test 16 | public void testCompare() throws Exception { 17 | BinlogFilePos first = BinlogFilePos.fromString("mysql-bin-changelog.218:14:6"); 18 | BinlogFilePos second = BinlogFilePos.fromString("mysql-bin-changelog.218:27:12"); 19 | BinlogFilePos third = BinlogFilePos.fromString("mysql-bin-changelog.219:11:92"); 20 | BinlogFilePos fourth = BinlogFilePos.fromString("mysql-bin-changelog.219:11:104"); 21 | 22 | assertTrue(first.compareTo(second) < 0); 23 | assertTrue(third.compareTo(second) > 0); 24 | assertTrue(third.compareTo(fourth) == 0); 25 | } 26 | 27 | @Test 28 | public void testCompareWithGTID() { 29 | String gtid1 = UUID1 + ":1-200"; 30 | String gtid2 = UUID1 + ":1-300"; 31 | String gtid3 = UUID1 + ":1-200," + UUID2 + ":1-456"; 32 | BinlogFilePos first = new BinlogFilePos("mysql-bin-changelog.218", 123, 456, gtid1, UUID1); 33 | BinlogFilePos second = new BinlogFilePos("mysql-bin-changelog.218", 456, 789, gtid2, UUID1); 34 | BinlogFilePos third = new BinlogFilePos("mysql-bin-changelog.100", 10, 24, gtid1, UUID2); 35 | BinlogFilePos fourth = new BinlogFilePos("mysql-bin-changelog.100", 20, 24, gtid3, UUID2); 36 | 37 | // server_uuid matches, compare binlog file number and position 38 | assertTrue(first.compareTo(second) < 0); 39 | 40 | // server_uuid doesn't match, compare GTID 41 | assertEquals(0, first.compareTo(third)); 42 | assertTrue(first.compareTo(fourth) < 0); 43 | assertTrue(second.compareTo(third) > 0); 44 | } 45 | 46 | @Test 47 | public void testConstructor() throws Exception { 48 | assertEquals( 49 | BinlogFilePos.fromString("mysql-bin-changelog.000218:14:6"), 50 | new BinlogFilePos("mysql-bin-changelog.000218", 14, 6)); 51 | 52 | assertEquals(new BinlogFilePos(80887L), new BinlogFilePos("mysql-bin-changelog.080887")); 53 | 54 | assertEquals(new BinlogFilePos(1080887L), new BinlogFilePos("mysql-bin-changelog.1080887")); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /spinaltap-model/src/test/java/com/airbnb/spinaltap/mysql/mutation/MysqlUpdateMutationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnDataType; 11 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 12 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 13 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 14 | import com.google.common.collect.ImmutableList; 15 | import com.google.common.collect.ImmutableMap; 16 | import org.junit.Test; 17 | 18 | public class MysqlUpdateMutationTest { 19 | @Test 20 | public void testAsColumnValues() throws Exception { 21 | Table table = 22 | new Table( 23 | 1, 24 | "table_name", 25 | "db_name", 26 | null, 27 | ImmutableList.of(new ColumnMetadata("id", ColumnDataType.LONGLONG, false, 0)), 28 | ImmutableList.of()); 29 | 30 | Row row = new Row(table, ImmutableMap.of("id", new Column(table.getColumns().get("id"), 2))); 31 | 32 | assertEquals(ImmutableMap.of("id", 2), MysqlUpdateMutation.asColumnValues(row)); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /spinaltap-model/src/test/java/com/airbnb/spinaltap/mysql/schema/RowTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import static org.junit.Assert.*; 8 | 9 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnDataType; 11 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 12 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 13 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 14 | import com.google.common.collect.ImmutableList; 15 | import com.google.common.collect.ImmutableMap; 16 | import org.junit.Test; 17 | 18 | public class RowTest { 19 | private static final int TABLE_ID = 1; 20 | private static final String TABLE_NAME = "Users"; 21 | private static final String DB_NAME = "test_db"; 22 | 23 | private static final String ID_COLUMN = "id"; 24 | private static final String NAME_COLUMN = "name"; 25 | 26 | @Test 27 | public void testNoPrimaryKey() throws Exception { 28 | Table table = 29 | new Table( 30 | TABLE_ID, 31 | TABLE_NAME, 32 | DB_NAME, 33 | null, 34 | ImmutableList.of(new ColumnMetadata(ID_COLUMN, ColumnDataType.LONGLONG, false, 0)), 35 | ImmutableList.of()); 36 | 37 | Row row = 38 | new Row( 39 | table, ImmutableMap.of(ID_COLUMN, new Column(table.getColumns().get(ID_COLUMN), 1))); 40 | 41 | assertNull(row.getPrimaryKeyValue()); 42 | } 43 | 44 | @Test 45 | public void testNullPrimaryKey() throws Exception { 46 | Table table = 47 | new Table( 48 | TABLE_ID, 49 | TABLE_NAME, 50 | DB_NAME, 51 | null, 52 | ImmutableList.of(new ColumnMetadata(ID_COLUMN, ColumnDataType.LONGLONG, true, 0)), 53 | ImmutableList.of(ID_COLUMN)); 54 | 55 | Row row = 56 | new Row( 57 | table, ImmutableMap.of(ID_COLUMN, new Column(table.getColumns().get(ID_COLUMN), null))); 58 | 59 | assertEquals("null", row.getPrimaryKeyValue()); 60 | } 61 | 62 | @Test 63 | public void testSinglePrimaryKey() throws Exception { 64 | Table table = 65 | new Table( 66 | TABLE_ID, 67 | TABLE_NAME, 68 | DB_NAME, 69 | null, 70 | ImmutableList.of( 71 | new ColumnMetadata(ID_COLUMN, ColumnDataType.LONGLONG, true, 0), 72 | new ColumnMetadata(NAME_COLUMN, ColumnDataType.VARCHAR, false, 1)), 73 | ImmutableList.of(ID_COLUMN)); 74 | 75 | Row row = 76 | new Row( 77 | table, 78 | ImmutableMap.of( 79 | ID_COLUMN, new Column(table.getColumns().get(ID_COLUMN), 1), 80 | NAME_COLUMN, new Column(table.getColumns().get(NAME_COLUMN), "Bob"))); 81 | 82 | assertEquals("1", row.getPrimaryKeyValue()); 83 | } 84 | 85 | @Test 86 | public void testCompositePrimaryKey() throws Exception { 87 | Table table = 88 | new Table( 89 | TABLE_ID, 90 | TABLE_NAME, 91 | DB_NAME, 92 | null, 93 | ImmutableList.of( 94 | new ColumnMetadata(ID_COLUMN, ColumnDataType.LONGLONG, true, 0), 95 | new ColumnMetadata(NAME_COLUMN, ColumnDataType.VARCHAR, true, 1)), 96 | ImmutableList.of(ID_COLUMN, NAME_COLUMN)); 97 | 98 | Row row = 99 | new Row( 100 | table, 101 | ImmutableMap.of( 102 | ID_COLUMN, new Column(table.getColumns().get(ID_COLUMN), 1), 103 | NAME_COLUMN, new Column(table.getColumns().get(NAME_COLUMN), "Bob"))); 104 | 105 | assertEquals("1Bob", row.getPrimaryKeyValue()); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /spinaltap-mysql/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'antlr' 2 | 3 | dependencies { 4 | compile project(':spinaltap-common') 5 | 6 | compile libraries.jdbi3 7 | compile libraries.mysql_binlog_connector 8 | compile libraries.guava_retrying 9 | compile libraries.jackson_datatype_guava 10 | compile libraries.jackson_dataformat_yaml 11 | compile libraries.mysql_connector 12 | compile libraries.hibernate_validator 13 | compileOnly libraries.lombok 14 | annotationProcessor libraries.lombok 15 | 16 | testCompile libraries.junit 17 | testCompile libraries.mokito 18 | testCompileOnly libraries.lombok 19 | testAnnotationProcessor libraries.lombok 20 | 21 | antlr libraries.antlr4 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/ColumnSerializationUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 8 | import java.io.Serializable; 9 | import java.nio.ByteBuffer; 10 | import java.util.Map; 11 | import lombok.NonNull; 12 | import lombok.experimental.UtilityClass; 13 | import lombok.extern.slf4j.Slf4j; 14 | import org.apache.commons.lang3.SerializationUtils; 15 | import org.apache.zookeeper.server.ByteBufferInputStream; 16 | 17 | /** A utility class for MySQL {@link Column} SerDe supoort. */ 18 | @Slf4j 19 | @UtilityClass 20 | public class ColumnSerializationUtil { 21 | public static byte[] serializeColumn(@NonNull final Column oldColumn) { 22 | return SerializationUtils.serialize(oldColumn.getValue()); 23 | } 24 | 25 | /** 26 | * mapping between column type to java type BIT => BitSet ENUM, YEAR TINY, SHORT, INT24, LONG => 27 | * int SET, LONGLONG => long FLOAT => float DOUBLE => value NEWDECIMAL => BigDecimal DATE => Date 28 | * TIME, TIME_V2 => Time TIMESTAMP, TIMESTAMP_V2 => Timestmap DATETIME, DATETIME_V2 => Date case 29 | * YEAR: STRING, VARCHAR, VAR_STRING => String BLOB => byte[] 30 | */ 31 | public static Serializable deserializeColumn( 32 | @NonNull final Map entity, @NonNull final String column) { 33 | final ByteBuffer byteBuffer = entity.get(column); 34 | 35 | if (byteBuffer == null) { 36 | return null; 37 | } 38 | 39 | final ByteBufferInputStream inputStream = new ByteBufferInputStream(byteBuffer); 40 | return (Serializable) SerializationUtils.deserialize(inputStream); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/MysqlDestinationMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import com.airbnb.common.metrics.TaggedMetricRegistry; 8 | import com.airbnb.spinaltap.Mutation; 9 | import com.airbnb.spinaltap.common.destination.DestinationMetrics; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutationMetadata; 11 | import com.google.common.base.Preconditions; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import lombok.NonNull; 15 | 16 | /** 17 | * Responsible for metrics collection on operations for {@link 18 | * com.airbnb.spinaltap.common.destination.Destination} and associated components for a given {@link 19 | * MysqlSource}. 20 | */ 21 | public class MysqlDestinationMetrics extends DestinationMetrics { 22 | private static final String DATABASE_NAME_TAG = "database_name"; 23 | private static final String TABLE_NAME_TAG = "table_name"; 24 | 25 | public MysqlDestinationMetrics( 26 | @NonNull final String sourceName, @NonNull final TaggedMetricRegistry metricRegistry) { 27 | this("mysql", sourceName, metricRegistry); 28 | } 29 | 30 | protected MysqlDestinationMetrics( 31 | @NonNull final String sourceType, 32 | @NonNull final String sourceName, 33 | @NonNull final TaggedMetricRegistry metricRegistry) { 34 | super(sourceName, sourceType, metricRegistry); 35 | } 36 | 37 | @Override 38 | protected Map getTags(@NonNull final Mutation.Metadata metadata) { 39 | Preconditions.checkState(metadata instanceof MysqlMutationMetadata); 40 | 41 | MysqlMutationMetadata mysqlMetadata = (MysqlMutationMetadata) metadata; 42 | Map metadataTags = new HashMap<>(); 43 | 44 | metadataTags.put(DATABASE_NAME_TAG, mysqlMetadata.getTable().getDatabase()); 45 | metadataTags.put(TABLE_NAME_TAG, mysqlMetadata.getTable().getName()); 46 | metadataTags.putAll(super.getTags(mysqlMetadata)); 47 | 48 | return metadataTags; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/StateRepository.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import com.airbnb.spinaltap.common.source.SourceState; 8 | import com.airbnb.spinaltap.common.util.Repository; 9 | import lombok.NonNull; 10 | import lombok.RequiredArgsConstructor; 11 | import lombok.extern.slf4j.Slf4j; 12 | 13 | /** Represents a repository for a {@link SourceState} record. */ 14 | @Slf4j 15 | @RequiredArgsConstructor 16 | public class StateRepository { 17 | @NonNull private final String sourceName; 18 | @NonNull private final Repository repository; 19 | @NonNull private final MysqlSourceMetrics metrics; 20 | 21 | /** Saves or updates the {@link SourceState} record in the repository */ 22 | public void save(@NonNull final S state) { 23 | try { 24 | repository.update( 25 | state, 26 | (currentValue, nextValue) -> { 27 | if (currentValue.getCurrentLeaderEpoch() > nextValue.getCurrentLeaderEpoch()) { 28 | log.warn("Will not update mysql state: current={}, next={}", currentValue, nextValue); 29 | return currentValue; 30 | } 31 | return nextValue; 32 | }); 33 | 34 | } catch (Exception ex) { 35 | log.error("Failed to save state for source " + sourceName, ex); 36 | metrics.stateSaveFailure(ex); 37 | throw new RuntimeException(ex); 38 | } 39 | 40 | log.info("Saved state for source {}. state={}", sourceName, state); 41 | metrics.stateSave(); 42 | } 43 | 44 | /** @return the {@link SourceState} record present in the repository. */ 45 | public S read() { 46 | S state = null; 47 | 48 | try { 49 | if (repository.exists()) { 50 | state = repository.get(); 51 | } else { 52 | log.info("State does not exist for source {}", sourceName); 53 | } 54 | } catch (Exception ex) { 55 | log.error("Failed to read state for source " + sourceName, ex); 56 | metrics.stateReadFailure(ex); 57 | throw new RuntimeException(ex); 58 | } 59 | 60 | log.debug("Read state for source {}. state={}", sourceName, state); 61 | metrics.stateRead(); 62 | return state; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/config/AbstractMysqlConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.config; 6 | 7 | import com.airbnb.spinaltap.common.config.DestinationConfiguration; 8 | import com.airbnb.spinaltap.common.config.SourceConfiguration; 9 | import com.airbnb.spinaltap.mysql.MysqlSource; 10 | import java.util.List; 11 | import lombok.NonNull; 12 | 13 | /** Represents the base configuration for a {@link MysqlSource}. */ 14 | public abstract class AbstractMysqlConfiguration extends SourceConfiguration { 15 | public AbstractMysqlConfiguration( 16 | @NonNull final String name, 17 | final String type, 18 | final String instanceTag, 19 | @NonNull final DestinationConfiguration destinationConfiguration) { 20 | super(name, type, instanceTag, destinationConfiguration); 21 | } 22 | 23 | public AbstractMysqlConfiguration(final String type, final String instanceTag) { 24 | super(type, instanceTag); 25 | } 26 | 27 | public abstract String getHost(); 28 | 29 | public abstract int getPort(); 30 | 31 | public abstract List getCanonicalTableNames(); 32 | 33 | public abstract String getOverridingDatabase(); 34 | } 35 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/config/MysqlSchemaStoreConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.config; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | import javax.validation.constraints.Max; 10 | import javax.validation.constraints.Min; 11 | import lombok.Data; 12 | import lombok.NoArgsConstructor; 13 | import lombok.NonNull; 14 | 15 | /** Represents the configuration for a {@link com.airbnb.spinaltap.mysql.schema.MysqlSchemaStore} */ 16 | @Data 17 | @NoArgsConstructor 18 | @JsonIgnoreProperties(ignoreUnknown = true) 19 | public class MysqlSchemaStoreConfiguration { 20 | @NonNull @JsonProperty private String host; 21 | 22 | @Min(0) 23 | @Max(65535) 24 | @JsonProperty 25 | private int port; 26 | 27 | @JsonProperty("mtls_enabled") 28 | private boolean mTlsEnabled; 29 | 30 | @NonNull @JsonProperty private String database = "schema_store"; 31 | 32 | @NonNull 33 | @JsonProperty("archive-database") 34 | private String archiveDatabase = "schema_store_archives"; 35 | } 36 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/BinlogEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.common.source.SourceEvent; 8 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 9 | import lombok.Getter; 10 | import lombok.ToString; 11 | 12 | /** Represents a Binlog event */ 13 | @Getter 14 | @ToString 15 | public abstract class BinlogEvent extends SourceEvent { 16 | private final long tableId; 17 | private final long serverId; 18 | private final BinlogFilePos binlogFilePos; 19 | 20 | public BinlogEvent(long tableId, long serverId, long timestamp, BinlogFilePos binlogFilePos) { 21 | super(timestamp); 22 | 23 | this.tableId = tableId; 24 | this.serverId = serverId; 25 | this.binlogFilePos = binlogFilePos; 26 | } 27 | 28 | public long getOffset() { 29 | return (binlogFilePos.getFileNumber() << 32) | binlogFilePos.getPosition(); 30 | } 31 | 32 | public boolean isMutation() { 33 | return this instanceof WriteEvent || this instanceof DeleteEvent || this instanceof UpdateEvent; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/DeleteEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import java.io.Serializable; 9 | import java.util.List; 10 | import lombok.Getter; 11 | 12 | @Getter 13 | public final class DeleteEvent extends BinlogEvent { 14 | private final List rows; 15 | 16 | public DeleteEvent( 17 | long tableId, 18 | long serverId, 19 | long timestamp, 20 | BinlogFilePos filePos, 21 | List rows) { 22 | super(tableId, serverId, timestamp, filePos); 23 | 24 | this.rows = rows; 25 | } 26 | 27 | @Override 28 | public int size() { 29 | return rows.size(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/GTIDEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import lombok.Getter; 9 | 10 | @Getter 11 | public class GTIDEvent extends BinlogEvent { 12 | private final String gtid; 13 | 14 | public GTIDEvent(long serverId, long timestamp, BinlogFilePos filePos, String gtid) { 15 | super(0, serverId, timestamp, filePos); 16 | this.gtid = gtid; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/QueryEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import lombok.Getter; 9 | 10 | @Getter 11 | public class QueryEvent extends BinlogEvent { 12 | private final String database; 13 | private final String sql; 14 | 15 | public QueryEvent( 16 | long serverId, long timestamp, BinlogFilePos filePos, String database, String sql) { 17 | super(0l, serverId, timestamp, filePos); 18 | 19 | this.database = database; 20 | this.sql = sql; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/StartEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | 9 | public class StartEvent extends BinlogEvent { 10 | public StartEvent(long serverId, long timestamp, BinlogFilePos filePos) { 11 | super(0L, serverId, timestamp, filePos); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/TableMapEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnDataType; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import lombok.Getter; 12 | 13 | @Getter 14 | public final class TableMapEvent extends BinlogEvent { 15 | private final String database; 16 | private final String table; 17 | private final List columnTypes; 18 | 19 | public TableMapEvent( 20 | long tableId, 21 | long serverId, 22 | long timestamp, 23 | BinlogFilePos filePos, 24 | String database, 25 | String table, 26 | byte[] columnTypeCodes) { 27 | super(tableId, serverId, timestamp, filePos); 28 | 29 | this.database = database; 30 | this.table = table; 31 | this.columnTypes = new ArrayList<>(); 32 | 33 | for (byte code : columnTypeCodes) { 34 | columnTypes.add(ColumnDataType.byCode(code)); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/UpdateEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import java.io.Serializable; 9 | import java.util.List; 10 | import java.util.Map; 11 | import lombok.Getter; 12 | 13 | @Getter 14 | public class UpdateEvent extends BinlogEvent { 15 | private final List> rows; 16 | 17 | public UpdateEvent( 18 | long tableId, 19 | long serverId, 20 | long timestamp, 21 | BinlogFilePos filePos, 22 | List> rows) { 23 | super(tableId, serverId, timestamp, filePos); 24 | 25 | this.rows = rows; 26 | } 27 | 28 | @Override 29 | public int size() { 30 | return rows.size(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/WriteEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import java.io.Serializable; 9 | import java.util.List; 10 | import lombok.Getter; 11 | 12 | @Getter 13 | public final class WriteEvent extends BinlogEvent { 14 | private final List rows; 15 | 16 | public WriteEvent( 17 | long tableId, 18 | long serverId, 19 | long timestamp, 20 | BinlogFilePos filePos, 21 | List rows) { 22 | super(tableId, serverId, timestamp, filePos); 23 | 24 | this.rows = rows; 25 | } 26 | 27 | @Override 28 | public int size() { 29 | return rows.size(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/XidEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import lombok.Getter; 9 | 10 | @Getter 11 | public class XidEvent extends BinlogEvent { 12 | private final long xid; 13 | 14 | public XidEvent(long serverId, long timestamp, BinlogFilePos filePos, long xid) { 15 | super(0l, serverId, timestamp, filePos); 16 | 17 | this.xid = xid; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/filter/DuplicateFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.filter; 6 | 7 | import com.airbnb.spinaltap.common.source.MysqlSourceState; 8 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 9 | import com.airbnb.spinaltap.mysql.GtidSet; 10 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 11 | import java.util.concurrent.atomic.AtomicReference; 12 | import lombok.NonNull; 13 | import lombok.RequiredArgsConstructor; 14 | 15 | /** 16 | * Represents a {@link com.airbnb.spinaltap.common.util.Filter} for duplicate {@link BinlogEvent}s 17 | * that have already been streamed. This is used for server-side de-duplication, by comparing 18 | * against the offset of the last marked {@link MysqlSourceState} checkpoint and disregarding any 19 | * events that are received with an offset before that watermark. 20 | */ 21 | @RequiredArgsConstructor 22 | public final class DuplicateFilter extends MysqlEventFilter { 23 | @NonNull private final AtomicReference state; 24 | 25 | public boolean apply(@NonNull final BinlogEvent event) { 26 | // Only applies to mutation events 27 | if (!event.isMutation()) { 28 | return true; 29 | } 30 | 31 | // We need to tell if position in `event` and in `state` are from the same source 32 | // MySQL server, because a failover may have happened and we are currently streaming 33 | // from the new master. 34 | // If they are from the same source server, we can just use the binlog filename and 35 | // position (offset) to tell whether we should skip this event. 36 | BinlogFilePos eventBinlogPos = event.getBinlogFilePos(); 37 | BinlogFilePos savedBinlogPos = state.get().getLastPosition(); 38 | // Use the same logic in BinlogFilePos.compareTo() here... 39 | if (BinlogFilePos.shouldCompareUsingFilePosition(eventBinlogPos, savedBinlogPos)) { 40 | return event.getOffset() > state.get().getLastOffset(); 41 | } 42 | 43 | // If this point is reached, a master failover might have happened. 44 | // We can only use GTIDSet to tell whether this event should be skipped. 45 | // We should only skip this event if GTIDSet in event is a "proper subset" of the GTIDSet 46 | // in saved state, because it is possible that the last transaction we streamed before the 47 | // failover is in the middle of a transaction. 48 | GtidSet eventGtidSet = eventBinlogPos.getGtidSet(); 49 | GtidSet savedGtidSet = savedBinlogPos.getGtidSet(); 50 | return !eventGtidSet.isContainedWithin(savedGtidSet) && !eventGtidSet.equals(savedGtidSet); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/filter/EventTypeFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.filter; 6 | 7 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 8 | import com.airbnb.spinaltap.mysql.event.DeleteEvent; 9 | import com.airbnb.spinaltap.mysql.event.GTIDEvent; 10 | import com.airbnb.spinaltap.mysql.event.QueryEvent; 11 | import com.airbnb.spinaltap.mysql.event.StartEvent; 12 | import com.airbnb.spinaltap.mysql.event.TableMapEvent; 13 | import com.airbnb.spinaltap.mysql.event.UpdateEvent; 14 | import com.airbnb.spinaltap.mysql.event.WriteEvent; 15 | import com.airbnb.spinaltap.mysql.event.XidEvent; 16 | import com.google.common.collect.ImmutableSet; 17 | import java.util.Set; 18 | import lombok.NonNull; 19 | import lombok.RequiredArgsConstructor; 20 | 21 | /** 22 | * Represents a {@link com.airbnb.spinaltap.common.util.Filter} for {@link BinlogEvent}s based on a 23 | * predefined whitelist of event class types. 24 | */ 25 | @RequiredArgsConstructor 26 | final class EventTypeFilter extends MysqlEventFilter { 27 | @SuppressWarnings("unchecked") 28 | private static final Set> WHITELISTED_EVENT_TYPES = 29 | ImmutableSet.of( 30 | TableMapEvent.class, 31 | WriteEvent.class, 32 | UpdateEvent.class, 33 | DeleteEvent.class, 34 | XidEvent.class, 35 | QueryEvent.class, 36 | StartEvent.class, 37 | GTIDEvent.class); 38 | 39 | public boolean apply(@NonNull final BinlogEvent event) { 40 | return WHITELISTED_EVENT_TYPES.contains(event.getClass()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/filter/MysqlEventFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.filter; 6 | 7 | import com.airbnb.spinaltap.common.source.MysqlSourceState; 8 | import com.airbnb.spinaltap.common.util.ChainedFilter; 9 | import com.airbnb.spinaltap.common.util.Filter; 10 | import com.airbnb.spinaltap.mysql.TableCache; 11 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 12 | import java.util.Set; 13 | import java.util.concurrent.atomic.AtomicReference; 14 | import lombok.NonNull; 15 | 16 | /** Base {@link com.airbnb.spinaltap.common.util.Filter} implement for MySQL {@link BinlogEvent}s */ 17 | public abstract class MysqlEventFilter implements Filter { 18 | public static Filter create( 19 | @NonNull final TableCache tableCache, 20 | @NonNull final Set tableNames, 21 | @NonNull final AtomicReference state) { 22 | return ChainedFilter.builder() 23 | .addFilter(new EventTypeFilter()) 24 | .addFilter(new TableFilter(tableCache, tableNames)) 25 | .addFilter(new DuplicateFilter(state)) 26 | .build(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/filter/TableFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.filter; 6 | 7 | import com.airbnb.spinaltap.mysql.TableCache; 8 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 9 | import com.airbnb.spinaltap.mysql.event.TableMapEvent; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 11 | import java.util.Set; 12 | import lombok.NonNull; 13 | import lombok.RequiredArgsConstructor; 14 | 15 | /** 16 | * Represents a {@link com.airbnb.spinaltap.common.util.Filter} for {@link BinlogEvent}s based on 17 | * the database table they belong to. This is used to ensure that mutations are propagated only for 18 | * events for the table the {@link com.airbnb.spinaltap.common.source.Source} is subscribed to. 19 | */ 20 | @RequiredArgsConstructor 21 | final class TableFilter extends MysqlEventFilter { 22 | @NonNull private final TableCache tableCache; 23 | @NonNull private final Set tableNames; 24 | 25 | public boolean apply(@NonNull final BinlogEvent event) { 26 | if (event instanceof TableMapEvent) { 27 | TableMapEvent tableMap = (TableMapEvent) event; 28 | return tableNames.contains( 29 | Table.canonicalNameOf(tableMap.getDatabase(), tableMap.getTable())); 30 | } else if (event.isMutation()) { 31 | return tableCache.contains(event.getTableId()); 32 | } 33 | 34 | return true; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/DeleteMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.mysql.DataSource; 8 | import com.airbnb.spinaltap.mysql.TableCache; 9 | import com.airbnb.spinaltap.mysql.Transaction; 10 | import com.airbnb.spinaltap.mysql.event.DeleteEvent; 11 | import com.airbnb.spinaltap.mysql.mutation.MysqlDeleteMutation; 12 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 13 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 14 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 15 | import java.io.Serializable; 16 | import java.util.ArrayList; 17 | import java.util.Collection; 18 | import java.util.List; 19 | import java.util.concurrent.atomic.AtomicLong; 20 | import java.util.concurrent.atomic.AtomicReference; 21 | import lombok.NonNull; 22 | 23 | /** 24 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} of a {@link DeleteEvent}s to the 25 | * corresponding list of {@link com.airbnb.spinaltap.mysql.mutation.MysqlMutation}s corresponding to 26 | * each row change in the event. 27 | */ 28 | final class DeleteMutationMapper extends MysqlMutationMapper { 29 | DeleteMutationMapper( 30 | @NonNull final DataSource dataSource, 31 | @NonNull final TableCache tableCache, 32 | @NonNull final AtomicReference beginTransaction, 33 | @NonNull final AtomicReference lastTransaction, 34 | @NonNull final AtomicLong leaderEpoch) { 35 | super(dataSource, tableCache, beginTransaction, lastTransaction, leaderEpoch); 36 | } 37 | 38 | @Override 39 | protected List mapEvent( 40 | @NonNull final Table table, @NonNull final DeleteEvent event) { 41 | final Collection cols = table.getColumns().values(); 42 | final List mutations = new ArrayList<>(); 43 | final List rows = event.getRows(); 44 | 45 | for (int position = 0; position < rows.size(); position++) { 46 | mutations.add( 47 | new MysqlDeleteMutation( 48 | createMetadata(table, event, position), 49 | new Row(table, zip(rows.get(position), cols)))); 50 | } 51 | 52 | return mutations; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/GTIDMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.common.util.Mapper; 8 | import com.airbnb.spinaltap.mysql.event.GTIDEvent; 9 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 10 | import java.util.Collections; 11 | import java.util.List; 12 | import java.util.concurrent.atomic.AtomicReference; 13 | import lombok.NonNull; 14 | import lombok.RequiredArgsConstructor; 15 | 16 | /** 17 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that keeps track of {@link 18 | * GTIDEvent}s, which will be included in {@link com.airbnb.spinaltap.mysql.Transaction} 19 | */ 20 | @RequiredArgsConstructor 21 | final class GTIDMapper implements Mapper> { 22 | private final AtomicReference gtid; 23 | 24 | @Override 25 | public List map(@NonNull final GTIDEvent event) { 26 | gtid.set(event.getGtid()); 27 | return Collections.emptyList(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/InsertMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.mysql.DataSource; 8 | import com.airbnb.spinaltap.mysql.TableCache; 9 | import com.airbnb.spinaltap.mysql.Transaction; 10 | import com.airbnb.spinaltap.mysql.event.WriteEvent; 11 | import com.airbnb.spinaltap.mysql.mutation.MysqlInsertMutation; 12 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 13 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 14 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 15 | import java.io.Serializable; 16 | import java.util.ArrayList; 17 | import java.util.Collection; 18 | import java.util.List; 19 | import java.util.concurrent.atomic.AtomicLong; 20 | import java.util.concurrent.atomic.AtomicReference; 21 | import lombok.NonNull; 22 | 23 | /** 24 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} of a {@link WriteEvent} to a list of 25 | * {@link com.airbnb.spinaltap.mysql.mutation.MysqlMutation}s corresponding to each row change in 26 | * the event. 27 | */ 28 | class InsertMutationMapper extends MysqlMutationMapper { 29 | InsertMutationMapper( 30 | @NonNull final DataSource dataSource, 31 | @NonNull final TableCache tableCache, 32 | @NonNull final AtomicReference beginTransaction, 33 | @NonNull final AtomicReference lastTransaction, 34 | @NonNull final AtomicLong leaderEpoch) { 35 | super(dataSource, tableCache, beginTransaction, lastTransaction, leaderEpoch); 36 | } 37 | 38 | @Override 39 | protected List mapEvent( 40 | @NonNull final Table table, @NonNull final WriteEvent event) { 41 | final List rows = event.getRows(); 42 | final List mutations = new ArrayList<>(); 43 | final Collection cols = table.getColumns().values(); 44 | 45 | for (int position = 0; position < rows.size(); position++) { 46 | mutations.add( 47 | new MysqlInsertMutation( 48 | createMetadata(table, event, position), 49 | new Row(table, zip(rows.get(position), cols)))); 50 | } 51 | 52 | return mutations; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/QueryMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.common.util.Mapper; 8 | import com.airbnb.spinaltap.mysql.Transaction; 9 | import com.airbnb.spinaltap.mysql.event.QueryEvent; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 11 | import com.airbnb.spinaltap.mysql.schema.MysqlSchemaManager; 12 | import java.util.Collections; 13 | import java.util.List; 14 | import java.util.concurrent.atomic.AtomicReference; 15 | import lombok.NonNull; 16 | import lombok.RequiredArgsConstructor; 17 | import lombok.extern.slf4j.Slf4j; 18 | 19 | /** 20 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that keeps track of {@link 21 | * QueryEvent}s. This is used to detect schema changes from DDL statements, and mark BEGIN 22 | * statements. 23 | */ 24 | @Slf4j 25 | @RequiredArgsConstructor 26 | final class QueryMapper implements Mapper> { 27 | private static final String BEGIN_STATEMENT = "BEGIN"; 28 | private static final String COMMIT_STATEMENT = "COMMIT"; 29 | 30 | private final AtomicReference beginTransaction; 31 | private final AtomicReference lastTransaction; 32 | private final AtomicReference gtid; 33 | private final MysqlSchemaManager schemaManager; 34 | 35 | public List map(@NonNull final QueryEvent event) { 36 | Transaction transaction = 37 | new Transaction( 38 | event.getTimestamp(), event.getOffset(), event.getBinlogFilePos(), gtid.get()); 39 | if (isTransactionBegin(event)) { 40 | beginTransaction.set(transaction); 41 | } else { 42 | // DDL is also a transaction 43 | lastTransaction.set(transaction); 44 | if (!isTransactionEnd(event)) { 45 | schemaManager.processDDL(event, gtid.get()); 46 | } 47 | } 48 | 49 | return Collections.emptyList(); 50 | } 51 | 52 | private boolean isTransactionBegin(final QueryEvent event) { 53 | return event.getSql().equals(BEGIN_STATEMENT); 54 | } 55 | 56 | private boolean isTransactionEnd(final QueryEvent event) { 57 | return event.getSql().equals(COMMIT_STATEMENT); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/StartMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.common.util.Mapper; 8 | import com.airbnb.spinaltap.mysql.DataSource; 9 | import com.airbnb.spinaltap.mysql.MysqlSourceMetrics; 10 | import com.airbnb.spinaltap.mysql.TableCache; 11 | import com.airbnb.spinaltap.mysql.event.StartEvent; 12 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 13 | import java.util.Collections; 14 | import java.util.List; 15 | import lombok.NonNull; 16 | import lombok.RequiredArgsConstructor; 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.joda.time.DateTime; 19 | 20 | /** 21 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that keeps track of binlog file 22 | * starts detected on {@link StartEvent}s. This is used to clear the {@link TableCache}, to ensure 23 | * table to tableId mapping remains consistent. 24 | */ 25 | @Slf4j 26 | @RequiredArgsConstructor 27 | final class StartMapper implements Mapper> { 28 | @NonNull private final DataSource dataSource; 29 | @NonNull private final TableCache tableCache; 30 | @NonNull private final MysqlSourceMetrics metrics; 31 | 32 | public List map(@NonNull final StartEvent event) { 33 | log.info( 34 | "Started processing binlog file {} for host {} at {}", 35 | event.getBinlogFilePos().getFileName(), 36 | dataSource.getHost(), 37 | new DateTime(event.getTimestamp())); 38 | 39 | metrics.binlogFileStart(); 40 | 41 | tableCache.clear(); 42 | return Collections.emptyList(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/TableMapMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.common.util.Mapper; 8 | import com.airbnb.spinaltap.mysql.TableCache; 9 | import com.airbnb.spinaltap.mysql.event.TableMapEvent; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 11 | import java.util.Collections; 12 | import java.util.List; 13 | import lombok.NonNull; 14 | import lombok.RequiredArgsConstructor; 15 | import lombok.extern.slf4j.Slf4j; 16 | 17 | /** 18 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that keeps track of {@link 19 | * com.airbnb.spinaltap.mysql.mutation.schema.Table} information from {@link TableMapEvent}s, which 20 | * will be appended as metadata to streamed {@link MysqlMutation}s. 21 | */ 22 | @Slf4j 23 | @RequiredArgsConstructor 24 | final class TableMapMapper implements Mapper> { 25 | @NonNull private final TableCache tableCache; 26 | 27 | /** 28 | * Updates the {@link TableCache} with {@link com.airbnb.spinaltap.mysql.mutation.schema.Table} 29 | * information corresponding to the {@link TableMapEvent}. To maintain consistency, any errors 30 | * will be propagated if the cache update fails. 31 | */ 32 | public List map(@NonNull final TableMapEvent event) { 33 | try { 34 | tableCache.addOrUpdate( 35 | event.getTableId(), event.getTable(), event.getDatabase(), event.getColumnTypes()); 36 | } catch (Exception ex) { 37 | log.error("Failed to process table map event: " + event, ex); 38 | throw new RuntimeException(ex); 39 | } 40 | 41 | return Collections.emptyList(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/UpdateMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.mysql.DataSource; 8 | import com.airbnb.spinaltap.mysql.TableCache; 9 | import com.airbnb.spinaltap.mysql.Transaction; 10 | import com.airbnb.spinaltap.mysql.event.UpdateEvent; 11 | import com.airbnb.spinaltap.mysql.mutation.MysqlDeleteMutation; 12 | import com.airbnb.spinaltap.mysql.mutation.MysqlInsertMutation; 13 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 14 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutationMetadata; 15 | import com.airbnb.spinaltap.mysql.mutation.MysqlUpdateMutation; 16 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 17 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 18 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 19 | import com.google.common.collect.Lists; 20 | import java.io.Serializable; 21 | import java.util.Collection; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.concurrent.atomic.AtomicLong; 25 | import java.util.concurrent.atomic.AtomicReference; 26 | import lombok.NonNull; 27 | 28 | /** 29 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} of a {@link UpdateEvent}s to the 30 | * corresponding list of {@link com.airbnb.spinaltap.mysql.mutation.MysqlMutation}s corresponding to 31 | * each row change in the event. 32 | */ 33 | final class UpdateMutationMapper extends MysqlMutationMapper { 34 | UpdateMutationMapper( 35 | @NonNull final DataSource dataSource, 36 | @NonNull final TableCache tableCache, 37 | @NonNull final AtomicReference beginTransaction, 38 | @NonNull final AtomicReference lastTransaction, 39 | @NonNull final AtomicLong leaderEpoch) { 40 | super(dataSource, tableCache, beginTransaction, lastTransaction, leaderEpoch); 41 | } 42 | 43 | @Override 44 | protected List mapEvent( 45 | @NonNull final Table table, @NonNull final UpdateEvent event) { 46 | final List mutations = Lists.newArrayList(); 47 | final Collection cols = table.getColumns().values(); 48 | final List> rows = event.getRows(); 49 | 50 | for (int position = 0; position < rows.size(); position++) { 51 | MysqlMutationMetadata metadata = createMetadata(table, event, position); 52 | 53 | final Row previousRow = new Row(table, zip(rows.get(position).getKey(), cols)); 54 | final Row newRow = new Row(table, zip(rows.get(position).getValue(), cols)); 55 | 56 | // If PK value has changed, then delete before image and insert new image 57 | // to retain invariant that a mutation captures changes to a single PK 58 | if (table.getPrimaryKey().isPresent() 59 | && !previousRow.getPrimaryKeyValue().equals(newRow.getPrimaryKeyValue())) { 60 | mutations.add(new MysqlDeleteMutation(metadata, previousRow)); 61 | mutations.add(new MysqlInsertMutation(metadata, newRow)); 62 | } else { 63 | mutations.add(new MysqlUpdateMutation(metadata, previousRow, newRow)); 64 | } 65 | } 66 | 67 | return mutations; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/event/mapper/XidMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.mapper; 6 | 7 | import com.airbnb.spinaltap.common.util.Mapper; 8 | import com.airbnb.spinaltap.mysql.MysqlSourceMetrics; 9 | import com.airbnb.spinaltap.mysql.Transaction; 10 | import com.airbnb.spinaltap.mysql.event.XidEvent; 11 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 12 | import java.util.Collections; 13 | import java.util.List; 14 | import java.util.concurrent.atomic.AtomicReference; 15 | import lombok.NonNull; 16 | import lombok.RequiredArgsConstructor; 17 | import lombok.extern.slf4j.Slf4j; 18 | 19 | /** 20 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that keeps track of {@link 21 | * Transaction} end information from {@link XidEvent}s, which will be appended as metadata to 22 | * streamed {@link MysqlMutation}s. 23 | */ 24 | @Slf4j 25 | @RequiredArgsConstructor 26 | final class XidMapper implements Mapper> { 27 | @NonNull private final AtomicReference endTransaction; 28 | @NonNull private final AtomicReference gtid; 29 | @NonNull private final MysqlSourceMetrics metrics; 30 | 31 | public List map(@NonNull final XidEvent event) { 32 | endTransaction.set( 33 | new Transaction( 34 | event.getTimestamp(), event.getOffset(), event.getBinlogFilePos(), gtid.get())); 35 | 36 | metrics.transactionReceived(); 37 | return Collections.emptyList(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/exception/InvalidBinlogPositionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.exception; 6 | 7 | import com.airbnb.spinaltap.common.exception.SpinaltapException; 8 | 9 | /** 10 | * Reflects that the binlog position set in the {@link com.airbnb.spinaltap.mysql.MysqlSource} 11 | * client is invalid. 12 | */ 13 | public class InvalidBinlogPositionException extends SpinaltapException { 14 | private static final long serialVersionUID = 9187451138457311547L; 15 | 16 | public InvalidBinlogPositionException(final String message) { 17 | super(message); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/mutation/MysqlKeyProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.util.KeyProvider; 9 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 11 | import com.google.common.base.Preconditions; 12 | import lombok.AccessLevel; 13 | import lombok.NoArgsConstructor; 14 | import lombok.NonNull; 15 | 16 | /** Represents a {@link KeyProvider} for {@link MysqlMutation}s. */ 17 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 18 | public class MysqlKeyProvider implements KeyProvider, String> { 19 | public static final MysqlKeyProvider INSTANCE = new MysqlKeyProvider(); 20 | 21 | /** 22 | * @return the key for a {@link MysqlMutation} in the following format: 23 | * "[database_name][table_name][primary_key_value]". 24 | */ 25 | @Override 26 | public String get(@NonNull final Mutation mutation) { 27 | Preconditions.checkState(mutation instanceof MysqlMutation); 28 | 29 | final MysqlMutation mysqlMutation = (MysqlMutation) mutation; 30 | final Table table = mysqlMutation.getMetadata().getTable(); 31 | final Row row = mysqlMutation.getRow(); 32 | 33 | return String.format( 34 | "%s:%s:%s", table.getDatabase(), table.getName(), row.getPrimaryKeyValue()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/mutation/mapper/DeleteMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.mapper; 6 | 7 | import com.airbnb.jitney.event.spinaltap.v1.Mutation; 8 | import com.airbnb.jitney.event.spinaltap.v1.MutationType; 9 | import com.airbnb.spinaltap.mysql.mutation.MysqlDeleteMutation; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutationMetadata; 11 | import lombok.NonNull; 12 | 13 | /** 14 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that maps a {@link 15 | * MysqlDeleteMutation} to its corresponding thrift {@link Mutation} form. 16 | */ 17 | class DeleteMutationMapper extends ThriftMutationMapper { 18 | public DeleteMutationMapper(final String sourceId) { 19 | super(sourceId); 20 | } 21 | 22 | public Mutation map(@NonNull final MysqlDeleteMutation mutation) { 23 | final MysqlMutationMetadata metadata = mutation.getMetadata(); 24 | 25 | return new Mutation( 26 | MutationType.DELETE, 27 | metadata.getTimestamp(), 28 | sourceId, 29 | metadata.getDataSource().getThriftDataSource(), 30 | createBinlogHeader(metadata, mutation.getType().getCode()), 31 | metadata.getTable().getThriftTable(), 32 | transformToEntity(mutation.getEntity())); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/mutation/mapper/InsertMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.mapper; 6 | 7 | import com.airbnb.jitney.event.spinaltap.v1.Mutation; 8 | import com.airbnb.jitney.event.spinaltap.v1.MutationType; 9 | import com.airbnb.spinaltap.mysql.mutation.MysqlInsertMutation; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutationMetadata; 11 | import lombok.NonNull; 12 | 13 | /** 14 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that maps a {@link 15 | * MysqlInsertMutation} to its corresponding thrift {@link Mutation} form. 16 | */ 17 | class InsertMutationMapper extends ThriftMutationMapper { 18 | public InsertMutationMapper(String sourceId) { 19 | super(sourceId); 20 | } 21 | 22 | public Mutation map(@NonNull final MysqlInsertMutation mutation) { 23 | final MysqlMutationMetadata metadata = mutation.getMetadata(); 24 | 25 | return new Mutation( 26 | MutationType.INSERT, 27 | metadata.getTimestamp(), 28 | sourceId, 29 | metadata.getDataSource().getThriftDataSource(), 30 | createBinlogHeader(metadata, mutation.getType().getCode()), 31 | metadata.getTable().getThriftTable(), 32 | transformToEntity(mutation.getEntity())); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/mutation/mapper/UpdateMutationMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation.mapper; 6 | 7 | import com.airbnb.jitney.event.spinaltap.v1.Mutation; 8 | import com.airbnb.jitney.event.spinaltap.v1.MutationType; 9 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutationMetadata; 10 | import com.airbnb.spinaltap.mysql.mutation.MysqlUpdateMutation; 11 | import lombok.NonNull; 12 | 13 | /** 14 | * Represents a {@link com.airbnb.spinaltap.common.util.Mapper} that maps a {@link 15 | * MysqlUpdateMutation} to its corresponding thrift {@link Mutation} form. 16 | */ 17 | class UpdateMutationMapper extends ThriftMutationMapper { 18 | public UpdateMutationMapper(final String sourceId) { 19 | super(sourceId); 20 | } 21 | 22 | public Mutation map(@NonNull final MysqlUpdateMutation mutation) { 23 | final MysqlMutationMetadata metadata = mutation.getMetadata(); 24 | 25 | final Mutation thriftMutation = 26 | new Mutation( 27 | MutationType.UPDATE, 28 | metadata.getTimestamp(), 29 | sourceId, 30 | metadata.getDataSource().getThriftDataSource(), 31 | createBinlogHeader(metadata, mutation.getType().getCode()), 32 | metadata.getTable().getThriftTable(), 33 | transformToEntity(mutation.getRow())); 34 | 35 | thriftMutation.setPreviousEntity(transformToEntity(mutation.getPreviousRow())); 36 | return thriftMutation; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/schema/MysqlColumn.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 8 | import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; 9 | import lombok.Builder; 10 | import lombok.RequiredArgsConstructor; 11 | import lombok.Value; 12 | 13 | @Value 14 | @Builder 15 | @RequiredArgsConstructor 16 | @JsonDeserialize(builder = MysqlColumn.MysqlColumnBuilder.class) 17 | public class MysqlColumn { 18 | String name; 19 | String dataType; 20 | String columnType; 21 | boolean primaryKey; 22 | 23 | @JsonPOJOBuilder(withPrefix = "") 24 | static class MysqlColumnBuilder {} 25 | } 26 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/schema/MysqlSchemaArchiver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | public interface MysqlSchemaArchiver { 8 | void archive(); 9 | } 10 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/schema/MysqlSchemaManagerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import com.airbnb.common.metrics.TaggedMetricRegistry; 8 | import com.airbnb.spinaltap.common.config.TlsConfiguration; 9 | import com.airbnb.spinaltap.mysql.MysqlClient; 10 | import com.airbnb.spinaltap.mysql.MysqlSourceMetrics; 11 | import com.airbnb.spinaltap.mysql.config.MysqlSchemaStoreConfiguration; 12 | import org.jdbi.v3.core.Jdbi; 13 | 14 | public class MysqlSchemaManagerFactory { 15 | private final String username; 16 | private final String password; 17 | private final MysqlSchemaStoreConfiguration configuration; 18 | private final TlsConfiguration tlsConfiguration; 19 | private Jdbi jdbi; 20 | 21 | public MysqlSchemaManagerFactory( 22 | final String username, 23 | final String password, 24 | final MysqlSchemaStoreConfiguration configuration, 25 | final TlsConfiguration tlsConfiguration) { 26 | this.username = username; 27 | this.password = password; 28 | this.configuration = configuration; 29 | this.tlsConfiguration = tlsConfiguration; 30 | 31 | if (configuration != null) { 32 | jdbi = 33 | Jdbi.create( 34 | MysqlClient.createMysqlDataSource( 35 | configuration.getHost(), 36 | configuration.getPort(), 37 | username, 38 | password, 39 | configuration.isMTlsEnabled(), 40 | tlsConfiguration)); 41 | jdbi.useHandle( 42 | handle -> { 43 | handle.execute( 44 | String.format("CREATE DATABASE IF NOT EXISTS `%s`", configuration.getDatabase())); 45 | handle.execute( 46 | String.format( 47 | "CREATE DATABASE IF NOT EXISTS `%s`", configuration.getArchiveDatabase())); 48 | }); 49 | } 50 | } 51 | 52 | public MysqlSchemaManager create( 53 | String sourceName, 54 | MysqlClient mysqlClient, 55 | boolean isSchemaVersionEnabled, 56 | MysqlSourceMetrics metrics) { 57 | MysqlSchemaReader schemaReader = 58 | new MysqlSchemaReader(sourceName, mysqlClient.getJdbi(), metrics); 59 | 60 | if (!isSchemaVersionEnabled) { 61 | return new MysqlSchemaManager(sourceName, null, null, schemaReader, mysqlClient, false); 62 | } 63 | 64 | MysqlSchemaStore schemaStore = 65 | new MysqlSchemaStore( 66 | sourceName, 67 | configuration.getDatabase(), 68 | configuration.getArchiveDatabase(), 69 | jdbi, 70 | metrics); 71 | MysqlSchemaDatabase schemaDatabase = new MysqlSchemaDatabase(sourceName, jdbi, metrics); 72 | return new MysqlSchemaManager( 73 | sourceName, schemaStore, schemaDatabase, schemaReader, mysqlClient, true); 74 | } 75 | 76 | public MysqlSchemaArchiver createArchiver(String sourceName) { 77 | MysqlSourceMetrics metrics = new MysqlSourceMetrics(sourceName, new TaggedMetricRegistry()); 78 | Jdbi jdbi = 79 | Jdbi.create( 80 | MysqlClient.createMysqlDataSource( 81 | configuration.getHost(), 82 | configuration.getPort(), 83 | username, 84 | password, 85 | configuration.isMTlsEnabled(), 86 | tlsConfiguration)); 87 | MysqlSchemaStore schemaStore = 88 | new MysqlSchemaStore( 89 | sourceName, 90 | configuration.getDatabase(), 91 | configuration.getArchiveDatabase(), 92 | jdbi, 93 | metrics); 94 | MysqlSchemaDatabase schemaDatabase = new MysqlSchemaDatabase(sourceName, jdbi, metrics); 95 | 96 | return new MysqlSchemaManager(sourceName, schemaStore, schemaDatabase, null, null, true); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/schema/MysqlSchemaUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import com.github.rholder.retry.Retryer; 8 | import com.github.rholder.retry.RetryerBuilder; 9 | import com.github.rholder.retry.StopStrategies; 10 | import com.github.rholder.retry.WaitStrategies; 11 | import java.sql.Connection; 12 | import java.sql.SQLException; 13 | import java.sql.Statement; 14 | import java.util.List; 15 | import java.util.concurrent.TimeUnit; 16 | import lombok.NonNull; 17 | import lombok.experimental.UtilityClass; 18 | import lombok.extern.slf4j.Slf4j; 19 | import org.jdbi.v3.core.Handle; 20 | import org.joda.time.DateTimeConstants; 21 | 22 | @Slf4j 23 | @UtilityClass 24 | public class MysqlSchemaUtil { 25 | public final Retryer VOID_RETRYER = createRetryer(); 26 | public final Retryer> LIST_COLUMN_RETRYER = createRetryer(); 27 | public final Retryer> LIST_TABLE_SCHEMA_RETRYER = createRetryer(); 28 | public final Retryer> LIST_STRING_RETRYER = createRetryer(); 29 | 30 | public void executeWithJdbc( 31 | @NonNull final Handle handle, final String database, @NonNull final String sql) 32 | throws SQLException { 33 | // Use JDBC API to excute raw SQL without any return value and no binding in SQL statement, so 34 | // we don't need to escape colon(:) 35 | // SQL statement with colon(:) inside needs to be escaped if using JDBI Handle.execute(sql) 36 | Connection connection = handle.getConnection(); 37 | if (database != null) { 38 | connection.setCatalog(database); 39 | } 40 | Statement statement = connection.createStatement(); 41 | statement.execute(sql); 42 | } 43 | 44 | private Retryer createRetryer() { 45 | return RetryerBuilder.newBuilder() 46 | .retryIfRuntimeException() 47 | .withWaitStrategy(WaitStrategies.exponentialWait(2, 30, TimeUnit.SECONDS)) 48 | .withStopStrategy(StopStrategies.stopAfterDelay(3 * DateTimeConstants.MILLIS_PER_MINUTE)) 49 | .build(); 50 | } 51 | 52 | public String escapeBackQuote(@NonNull final String name) { 53 | // MySQL allows backquote in database/table name, but need to escape it in DDL 54 | return name.replace("`", "``"); 55 | } 56 | 57 | String removeCommentsFromDDL(final String ddl) { 58 | return ddl 59 | // https://dev.mysql.com/doc/refman/5.7/en/comments.html 60 | // Replace MySQL-specific comments (/*! ... */ and /*!50110 ... */) which 61 | // are actually executed 62 | .replaceAll("/\\*!(?:\\d{5})?(.*?)\\*/", "$1") 63 | // Remove block comments 64 | // https://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment 65 | // line comments and newlines are kept 66 | // Note: This does not handle comments in quotes 67 | .replaceAll("/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*/", " ") 68 | // Remove extra spaces 69 | .replaceAll("\\h+", " ") 70 | .replaceAll("^\\s+", ""); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/schema/MysqlTableSchema.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 8 | import java.util.List; 9 | import java.util.Map; 10 | import lombok.Value; 11 | 12 | @Value 13 | public class MysqlTableSchema { 14 | long id; 15 | String database; 16 | String table; 17 | BinlogFilePos binlogFilePos; 18 | String gtid; 19 | String sql; 20 | long timestamp; 21 | List columns; 22 | Map metadata; 23 | } 24 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/validator/EventOrderValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.validator; 6 | 7 | import com.airbnb.spinaltap.common.util.Validator; 8 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 9 | import java.util.function.Consumer; 10 | import lombok.NonNull; 11 | import lombok.RequiredArgsConstructor; 12 | import lombok.extern.slf4j.Slf4j; 13 | 14 | /** 15 | * Represents a {@link Validator} that asserts {@link BinlogEvent}s are streamed in order of event 16 | * id (offset). The implement assumes {@code validate} is called on events in the order they are 17 | * received. 18 | */ 19 | @Slf4j 20 | @RequiredArgsConstructor 21 | public class EventOrderValidator implements Validator { 22 | /** The handler to call on {@link BinlogEvent}s that are out of order. */ 23 | @NonNull private final Consumer handler; 24 | 25 | private long lastSeenId = -1; 26 | 27 | @Override 28 | public void validate(@NonNull final BinlogEvent event) { 29 | long eventId = event.getOffset(); 30 | log.debug("Validating order for event with id {}. {}", eventId, event); 31 | 32 | if (eventId > 0 && lastSeenId > eventId) { 33 | log.warn( 34 | "Mutation with id {} is out of order and should precede {}. {}", 35 | eventId, 36 | lastSeenId, 37 | event); 38 | handler.accept(event); 39 | } 40 | 41 | lastSeenId = eventId; 42 | } 43 | 44 | @Override 45 | public void reset() { 46 | lastSeenId = -1; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/main/java/com/airbnb/spinaltap/mysql/validator/MutationSchemaValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.validator; 6 | 7 | import com.airbnb.spinaltap.Mutation; 8 | import com.airbnb.spinaltap.common.util.Validator; 9 | import com.airbnb.spinaltap.mysql.mutation.MysqlMutation; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 11 | import java.util.Map; 12 | import java.util.function.Consumer; 13 | import java.util.stream.Collectors; 14 | import lombok.NonNull; 15 | import lombok.RequiredArgsConstructor; 16 | import lombok.extern.slf4j.Slf4j; 17 | 18 | /** 19 | * Represents a {@link Validator} that asserts parity of the {@link 20 | * com.airbnb.spinaltap.mysql.mutation.schema.Table} schema with the {@link 21 | * com.airbnb.spinaltap.mysql.mutation.schema.Column} schema associated with a {@link MysqlMutation} 22 | */ 23 | @Slf4j 24 | @RequiredArgsConstructor 25 | public final class MutationSchemaValidator implements Validator { 26 | /** The handler to call on {@link Mutation}s that are invalid. */ 27 | @NonNull private final Consumer> handler; 28 | 29 | @Override 30 | public void validate(@NonNull final MysqlMutation mutation) { 31 | log.debug("Validating schema for mutation: {}", mutation); 32 | 33 | if (!hasValidSchema(mutation.getRow())) { 34 | log.warn("Invalid schema detected for mutation: {}", mutation); 35 | handler.accept(mutation); 36 | } 37 | } 38 | 39 | private boolean hasValidSchema(final Row row) { 40 | return row.getColumns() 41 | .entrySet() 42 | .stream() 43 | .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getMetadata())) 44 | .equals(row.getTable().getColumns()); 45 | } 46 | 47 | @Override 48 | public void reset() {} 49 | } 50 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/ColumnSerializationUtilTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import com.airbnb.jitney.event.spinaltap.v1.BinlogHeader; 10 | import com.airbnb.jitney.event.spinaltap.v1.DataSource; 11 | import com.airbnb.jitney.event.spinaltap.v1.Mutation; 12 | import com.airbnb.jitney.event.spinaltap.v1.MutationType; 13 | import com.airbnb.jitney.event.spinaltap.v1.Table; 14 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 15 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnDataType; 16 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 17 | import com.google.common.collect.ImmutableMap; 18 | import com.google.common.collect.ImmutableSet; 19 | import java.nio.ByteBuffer; 20 | import java.util.Map; 21 | import org.apache.thrift.TDeserializer; 22 | import org.apache.thrift.TSerializer; 23 | import org.apache.thrift.protocol.TBinaryProtocol; 24 | import org.junit.Test; 25 | 26 | public class ColumnSerializationUtilTest { 27 | private static final long TIMESTAMP = 1234; 28 | private static final String SOURCE_ID = "localhost"; 29 | private static final BinlogHeader BINLOG_HEADER = new BinlogHeader("123", 2, 3, 4); 30 | private static final DataSource DATA_SOURCE = new DataSource("localhost", 9192, "db"); 31 | private static final Table TABLE = 32 | new Table( 33 | TIMESTAMP, 34 | "table", 35 | "db", 36 | ImmutableSet.of("c1", "c2"), 37 | ImmutableMap.of("c1", new com.airbnb.jitney.event.spinaltap.v1.Column(1, false, "c1"))); 38 | 39 | @Test 40 | public void testDeserializeColumn() throws Exception { 41 | Mutation mutation = 42 | new Mutation( 43 | MutationType.DELETE, 44 | TIMESTAMP, 45 | SOURCE_ID, 46 | DATA_SOURCE, 47 | BINLOG_HEADER, 48 | TABLE, 49 | getEntity()); 50 | 51 | TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); 52 | TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); 53 | 54 | byte[] serialized = serializer.serialize(mutation); 55 | 56 | Mutation deserialized = new Mutation(); 57 | deserializer.deserialize(deserialized, serialized); 58 | 59 | assertEquals(mutation, deserialized); 60 | } 61 | 62 | private static Map getEntity() { 63 | return ImmutableMap.of( 64 | "c1", 65 | ByteBuffer.wrap( 66 | ColumnSerializationUtil.serializeColumn( 67 | new Column(new ColumnMetadata("c1", ColumnDataType.INT24, false, 0), 12345))), 68 | "c2", 69 | ByteBuffer.wrap( 70 | ColumnSerializationUtil.serializeColumn( 71 | new Column(new ColumnMetadata("c2", ColumnDataType.STRING, false, 1), "string"))), 72 | "c3", 73 | ByteBuffer.wrap( 74 | ColumnSerializationUtil.serializeColumn( 75 | new Column( 76 | new ColumnMetadata("c3", ColumnDataType.BLOB, false, 2), 77 | "blob.data".getBytes()))), 78 | "c4", 79 | ByteBuffer.wrap( 80 | ColumnSerializationUtil.serializeColumn( 81 | new Column(new ColumnMetadata("c4", ColumnDataType.DATETIME, false, 3), null)))); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/EventOrderValidatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql; 6 | 7 | import static org.junit.Assert.*; 8 | import static org.mockito.Mockito.*; 9 | 10 | import com.airbnb.spinaltap.common.source.SourceEvent; 11 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 12 | import com.airbnb.spinaltap.mysql.validator.EventOrderValidator; 13 | import com.google.common.collect.Lists; 14 | import java.util.Collections; 15 | import java.util.List; 16 | import org.junit.Test; 17 | 18 | public class EventOrderValidatorTest { 19 | private final BinlogEvent firstEvent = mock(BinlogEvent.class); 20 | private final BinlogEvent secondEvent = mock(BinlogEvent.class); 21 | 22 | @Test 23 | public void testEventInOrder() throws Exception { 24 | List unorderedEvents = Lists.newArrayList(); 25 | 26 | when(firstEvent.getOffset()).thenReturn(1L); 27 | when(secondEvent.getOffset()).thenReturn(2L); 28 | 29 | EventOrderValidator validator = new EventOrderValidator(unorderedEvents::add); 30 | 31 | validator.validate(firstEvent); 32 | validator.validate(secondEvent); 33 | 34 | assertTrue(unorderedEvents.isEmpty()); 35 | } 36 | 37 | @Test 38 | public void testEventOutOfOrder() throws Exception { 39 | List unorderedEvents = Lists.newArrayList(); 40 | 41 | when(firstEvent.getOffset()).thenReturn(2L); 42 | when(secondEvent.getOffset()).thenReturn(1L); 43 | 44 | EventOrderValidator validator = new EventOrderValidator(unorderedEvents::add); 45 | 46 | validator.validate(firstEvent); 47 | validator.validate(secondEvent); 48 | 49 | assertEquals(Collections.singletonList(secondEvent), unorderedEvents); 50 | } 51 | 52 | @Test 53 | public void testReset() throws Exception { 54 | List unorderedEvents = Lists.newArrayList(); 55 | 56 | when(firstEvent.getOffset()).thenReturn(1L); 57 | when(secondEvent.getOffset()).thenReturn(2L); 58 | 59 | EventOrderValidator validator = new EventOrderValidator(unorderedEvents::add); 60 | 61 | validator.validate(firstEvent); 62 | validator.validate(secondEvent); 63 | 64 | validator.reset(); 65 | 66 | validator.validate(firstEvent); 67 | validator.validate(secondEvent); 68 | 69 | assertTrue(unorderedEvents.isEmpty()); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/event/filter/MysqlEventFilterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.event.filter; 6 | 7 | import static org.junit.Assert.*; 8 | import static org.mockito.Mockito.*; 9 | 10 | import com.airbnb.spinaltap.common.source.MysqlSourceState; 11 | import com.airbnb.spinaltap.common.util.Filter; 12 | import com.airbnb.spinaltap.mysql.BinlogFilePos; 13 | import com.airbnb.spinaltap.mysql.TableCache; 14 | import com.airbnb.spinaltap.mysql.event.BinlogEvent; 15 | import com.airbnb.spinaltap.mysql.event.DeleteEvent; 16 | import com.airbnb.spinaltap.mysql.event.QueryEvent; 17 | import com.airbnb.spinaltap.mysql.event.StartEvent; 18 | import com.airbnb.spinaltap.mysql.event.TableMapEvent; 19 | import com.airbnb.spinaltap.mysql.event.UpdateEvent; 20 | import com.airbnb.spinaltap.mysql.event.WriteEvent; 21 | import com.airbnb.spinaltap.mysql.event.XidEvent; 22 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 23 | import com.google.common.collect.Sets; 24 | import java.util.Collections; 25 | import java.util.Set; 26 | import java.util.concurrent.atomic.AtomicReference; 27 | import org.junit.Test; 28 | 29 | public class MysqlEventFilterTest { 30 | private static final String DATABASE_NAME = "db"; 31 | private static final String TABLE_NAME = "users"; 32 | private static final long TABLE_ID = 1l; 33 | private static final Set TABLE_NAMES = 34 | Sets.newHashSet(Table.canonicalNameOf(DATABASE_NAME, TABLE_NAME)); 35 | private static final BinlogFilePos BINLOG_FILE_POS = new BinlogFilePos("test.123", 14, 100); 36 | 37 | @Test 38 | public void testEventFilter() throws Exception { 39 | TableCache tableCache = mock(TableCache.class); 40 | BinlogEvent lastEvent = new XidEvent(0l, 0l, BINLOG_FILE_POS, 0l); 41 | BinlogFilePos nextPosition = new BinlogFilePos("test.123", 15, 100); 42 | MysqlSourceState state = new MysqlSourceState(0l, lastEvent.getOffset(), 0l, BINLOG_FILE_POS); 43 | Filter filter = 44 | MysqlEventFilter.create(tableCache, TABLE_NAMES, new AtomicReference(state)); 45 | 46 | when(tableCache.contains(TABLE_ID)).thenReturn(true); 47 | 48 | assertTrue( 49 | filter.apply( 50 | new TableMapEvent( 51 | TABLE_ID, 0l, 0l, nextPosition, DATABASE_NAME, TABLE_NAME, new byte[1]))); 52 | assertTrue( 53 | filter.apply(new WriteEvent(TABLE_ID, 0l, 0l, nextPosition, Collections.emptyList()))); 54 | assertTrue( 55 | filter.apply(new DeleteEvent(TABLE_ID, 0l, 0l, nextPosition, Collections.emptyList()))); 56 | assertTrue( 57 | filter.apply(new UpdateEvent(TABLE_ID, 0l, 0l, nextPosition, Collections.emptyList()))); 58 | assertTrue(filter.apply(new XidEvent(0l, 0l, BINLOG_FILE_POS, 12l))); 59 | assertTrue(filter.apply(new QueryEvent(0l, 0l, BINLOG_FILE_POS, DATABASE_NAME, ""))); 60 | assertTrue(filter.apply(new StartEvent(0l, 0l, BINLOG_FILE_POS))); 61 | 62 | assertFalse( 63 | filter.apply(new TableMapEvent(TABLE_ID, 0l, 0l, BINLOG_FILE_POS, "", "", new byte[1]))); 64 | assertFalse(filter.apply(new WriteEvent(0l, 0l, 0l, BINLOG_FILE_POS, Collections.emptyList()))); 65 | assertFalse( 66 | filter.apply(new WriteEvent(TABLE_ID, 0l, 0l, BINLOG_FILE_POS, Collections.emptyList()))); 67 | assertFalse(filter.apply(mock(BinlogEvent.class))); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/mutation/MysqlKeyProviderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.mutation; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import com.airbnb.spinaltap.mysql.mutation.schema.Column; 10 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnDataType; 11 | import com.airbnb.spinaltap.mysql.mutation.schema.ColumnMetadata; 12 | import com.airbnb.spinaltap.mysql.mutation.schema.Row; 13 | import com.airbnb.spinaltap.mysql.mutation.schema.Table; 14 | import com.google.common.collect.ImmutableList; 15 | import com.google.common.collect.ImmutableMap; 16 | import org.junit.Test; 17 | 18 | public class MysqlKeyProviderTest { 19 | private static final String ID_COLUMN = "id"; 20 | 21 | private static final Table TABLE = 22 | new Table( 23 | 0L, 24 | "users", 25 | "test", 26 | null, 27 | ImmutableList.of(new ColumnMetadata(ID_COLUMN, ColumnDataType.LONGLONG, true, 0)), 28 | ImmutableList.of(ID_COLUMN)); 29 | 30 | private static final MysqlMutationMetadata MUTATION_METADATA = 31 | new MysqlMutationMetadata(null, null, TABLE, 0L, 0L, 0L, null, null, 0L, 0); 32 | 33 | @Test 34 | public void testGetKey() throws Exception { 35 | Row row = 36 | new Row( 37 | TABLE, ImmutableMap.of(ID_COLUMN, new Column(TABLE.getColumns().get(ID_COLUMN), 1234))); 38 | MysqlMutation mutation = new MysqlInsertMutation(MUTATION_METADATA, row); 39 | 40 | assertEquals("test:users:1234", MysqlKeyProvider.INSTANCE.get(mutation)); 41 | } 42 | 43 | @Test 44 | public void testGetNullKey() throws Exception { 45 | Row row = 46 | new Row( 47 | TABLE, ImmutableMap.of(ID_COLUMN, new Column(TABLE.getColumns().get(ID_COLUMN), null))); 48 | MysqlMutation mutation = new MysqlInsertMutation(MUTATION_METADATA, row); 49 | 50 | assertEquals("test:users:null", MysqlKeyProvider.INSTANCE.get(mutation)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/schema/MysqlColumnTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import static org.junit.Assert.*; 8 | 9 | import com.fasterxml.jackson.core.type.TypeReference; 10 | import com.fasterxml.jackson.databind.ObjectMapper; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import org.junit.Test; 14 | 15 | public class MysqlColumnTest { 16 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 17 | private static final MysqlColumn COLUMN = 18 | new MysqlColumn("column1", "varchar", "varchar(255)", false); 19 | private static final List COLUMNS = 20 | Arrays.asList( 21 | new MysqlColumn("id", "int", "int(20)", true), 22 | COLUMN, 23 | new MysqlColumn("column2", "text", "text", false)); 24 | 25 | @Test 26 | public void testJSONSerDer() throws Exception { 27 | String jsonString = OBJECT_MAPPER.writeValueAsString(COLUMN); 28 | MysqlColumn deserialized = OBJECT_MAPPER.readValue(jsonString, MysqlColumn.class); 29 | assertEquals(COLUMN, deserialized); 30 | 31 | jsonString = OBJECT_MAPPER.writeValueAsString(COLUMNS); 32 | List deserializedColumns = 33 | OBJECT_MAPPER.readValue(jsonString, new TypeReference>() {}); 34 | assertEquals(COLUMNS, deserializedColumns); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /spinaltap-mysql/src/test/java/com/airbnb/spinaltap/mysql/schema/MysqlSchemaUtilTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap.mysql.schema; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | import org.junit.Test; 10 | 11 | public class MysqlSchemaUtilTest { 12 | @Test 13 | public void testBlockSQLCommentsRemoval() { 14 | String sql_with_block_comments = 15 | "CREATE/* ! COMMENTS ! */UNIQUE /* ANOTHER COMMENTS ! */INDEX unique_index\n" 16 | + "ON `my_db`.`my_table` (`col1`, `col2`)"; 17 | String sql_with_comments_in_multi_lines = 18 | "CREATE UNIQUE /*\n" 19 | + "COMMENT Line1 \n" 20 | + "COMMENT Line 2\n" 21 | + "*/\n" 22 | + "INDEX ON `my_db`.`my_table` (`col1`, `col2`)"; 23 | String expected_sql = 24 | "CREATE UNIQUE INDEX unique_index\nON `my_db`.`my_table` (`col1`, `col2`)"; 25 | String stripped_sql = MysqlSchemaUtil.removeCommentsFromDDL(sql_with_block_comments); 26 | assertEquals(expected_sql, stripped_sql); 27 | 28 | stripped_sql = MysqlSchemaUtil.removeCommentsFromDDL(sql_with_comments_in_multi_lines); 29 | expected_sql = "CREATE UNIQUE \nINDEX ON `my_db`.`my_table` (`col1`, `col2`)"; 30 | assertEquals(expected_sql, stripped_sql); 31 | } 32 | 33 | @Test 34 | public void testMySQLSpecCommentsRemoval() { 35 | String sql_with_mysql_spec_comments = 36 | "CREATE TABLE t1(a INT, KEY (a)) /*!50110 KEY_BLOCK_SIZE=1024 */"; 37 | String sql_with_mysql_spec_comments2 = "/*!CREATE TABLE t1(a INT, KEY (a))*/"; 38 | 39 | String expected_sql = "CREATE TABLE t1(a INT, KEY (a)) KEY_BLOCK_SIZE=1024 "; 40 | String stripped_sql = MysqlSchemaUtil.removeCommentsFromDDL(sql_with_mysql_spec_comments); 41 | assertEquals(expected_sql, stripped_sql); 42 | 43 | expected_sql = "CREATE TABLE t1(a INT, KEY (a))"; 44 | stripped_sql = MysqlSchemaUtil.removeCommentsFromDDL(sql_with_mysql_spec_comments2); 45 | assertEquals(expected_sql, stripped_sql); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /spinaltap-standalone/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "com.github.johnrengelman.shadow" version "5.0.0" 3 | } 4 | 5 | dependencies { 6 | compile project(':spinaltap-mysql') 7 | compile project(':spinaltap-kafka') 8 | compileOnly libraries.lombok 9 | annotationProcessor libraries.lombok 10 | } 11 | 12 | jar { 13 | manifest { 14 | attributes 'Main-Class': 'com.airbnb.spinaltap.SpinalTapStandaloneApp' 15 | attributes 'Description': 'SpinalTap Standalone Application' 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /spinaltap-standalone/src/main/java/com/airbnb/spinaltap/SpinalTapStandaloneApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import com.airbnb.common.metrics.TaggedMetricRegistry; 8 | import com.airbnb.spinaltap.common.pipe.PipeManager; 9 | import com.airbnb.spinaltap.kafka.KafkaDestinationBuilder; 10 | import com.airbnb.spinaltap.mysql.MysqlPipeFactory; 11 | import com.airbnb.spinaltap.mysql.config.MysqlConfiguration; 12 | import com.airbnb.spinaltap.mysql.schema.MysqlSchemaManagerFactory; 13 | import com.fasterxml.jackson.databind.ObjectMapper; 14 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 15 | import com.google.common.collect.ImmutableMap; 16 | import java.io.File; 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.apache.curator.framework.CuratorFramework; 19 | import org.apache.curator.framework.CuratorFrameworkFactory; 20 | import org.apache.curator.retry.ExponentialBackoffRetry; 21 | 22 | /** A standalone single-node application to run SpinalTap process. */ 23 | @Slf4j 24 | public final class SpinalTapStandaloneApp { 25 | public static void main(String[] args) throws Exception { 26 | if (args.length != 1) { 27 | log.error("Usage: SpinalTapStandaloneApp "); 28 | System.exit(1); 29 | } 30 | 31 | final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); 32 | final SpinalTapStandaloneConfiguration config = 33 | objectMapper.readValue(new File(args[0]), SpinalTapStandaloneConfiguration.class); 34 | 35 | final MysqlPipeFactory mysqlPipeFactory = createMysqlPipeFactory(config); 36 | final ZookeeperRepositoryFactory zkRepositoryFactory = createZookeeperRepositoryFactory(config); 37 | final PipeManager pipeManager = new PipeManager(); 38 | 39 | for (MysqlConfiguration mysqlSourceConfig : config.getMysqlSources()) { 40 | final String sourceName = mysqlSourceConfig.getName(); 41 | final String partitionName = String.format("%s_0", sourceName); 42 | pipeManager.addPipes( 43 | sourceName, 44 | partitionName, 45 | mysqlPipeFactory.createPipes(mysqlSourceConfig, partitionName, zkRepositoryFactory, 0)); 46 | } 47 | 48 | Runtime.getRuntime().addShutdownHook(new Thread(pipeManager::stop)); 49 | } 50 | 51 | private static MysqlPipeFactory createMysqlPipeFactory( 52 | final SpinalTapStandaloneConfiguration config) { 53 | return new MysqlPipeFactory( 54 | config.getMysqlUser(), 55 | config.getMysqlPassword(), 56 | config.getMysqlServerId(), 57 | config.getTlsConfiguration(), 58 | ImmutableMap.of( 59 | "kafka", () -> new KafkaDestinationBuilder<>(config.getKafkaProducerConfig())), 60 | new MysqlSchemaManagerFactory( 61 | config.getMysqlUser(), 62 | config.getMysqlPassword(), 63 | config.getMysqlSchemaStoreConfig(), 64 | config.getTlsConfiguration()), 65 | new TaggedMetricRegistry()); 66 | } 67 | 68 | private static ZookeeperRepositoryFactory createZookeeperRepositoryFactory( 69 | final SpinalTapStandaloneConfiguration config) { 70 | final CuratorFramework zkClient = 71 | CuratorFrameworkFactory.builder() 72 | .namespace(config.getZkNamespace()) 73 | .connectString(config.getZkConnectionString()) 74 | .retryPolicy(new ExponentialBackoffRetry(100, 3)) 75 | .build(); 76 | 77 | zkClient.start(); 78 | 79 | return new ZookeeperRepositoryFactory(zkClient); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /spinaltap-standalone/src/main/java/com/airbnb/spinaltap/SpinalTapStandaloneConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import com.airbnb.spinaltap.common.config.TlsConfiguration; 8 | import com.airbnb.spinaltap.kafka.KafkaProducerConfiguration; 9 | import com.airbnb.spinaltap.mysql.config.MysqlConfiguration; 10 | import com.airbnb.spinaltap.mysql.config.MysqlSchemaStoreConfiguration; 11 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 12 | import com.fasterxml.jackson.annotation.JsonProperty; 13 | import java.util.List; 14 | import javax.validation.constraints.NotNull; 15 | import lombok.Data; 16 | 17 | /** Represents the {@link SpinalTapStandaloneApp} configuration. */ 18 | @Data 19 | @JsonIgnoreProperties(ignoreUnknown = true) 20 | public class SpinalTapStandaloneConfiguration { 21 | public static final int DEFAULT_MYSQL_SERVER_ID = 65535; 22 | 23 | @NotNull 24 | @JsonProperty("zk-connection-string") 25 | private String zkConnectionString; 26 | 27 | @NotNull 28 | @JsonProperty("zk-namespace") 29 | private String zkNamespace; 30 | 31 | @NotNull 32 | @JsonProperty("kafka-config") 33 | private KafkaProducerConfiguration kafkaProducerConfig; 34 | 35 | /** 36 | * Note: The user should have following grants on the source databases: 37 | * 38 | *

44 | */ 45 | @NotNull 46 | @JsonProperty("mysql-user") 47 | private String mysqlUser; 48 | 49 | @NotNull 50 | @JsonProperty("mysql-password") 51 | private String mysqlPassword; 52 | 53 | @NotNull 54 | @JsonProperty("mysql-server-id") 55 | private long mysqlServerId = DEFAULT_MYSQL_SERVER_ID; 56 | 57 | @JsonProperty("tls-config") 58 | private TlsConfiguration tlsConfiguration; 59 | 60 | @JsonProperty("mysql-schema-store") 61 | private MysqlSchemaStoreConfiguration mysqlSchemaStoreConfig; 62 | 63 | @NotNull 64 | @JsonProperty("mysql-sources") 65 | private List mysqlSources; 66 | } 67 | -------------------------------------------------------------------------------- /spinaltap-standalone/src/main/java/com/airbnb/spinaltap/ZookeeperRepositoryFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Airbnb. Licensed under Apache-2.0. See License in the project root for license 3 | * information. 4 | */ 5 | package com.airbnb.spinaltap; 6 | 7 | import com.airbnb.spinaltap.common.source.MysqlSourceState; 8 | import com.airbnb.spinaltap.common.util.StateRepositoryFactory; 9 | import com.airbnb.spinaltap.common.util.ZookeeperRepository; 10 | import com.fasterxml.jackson.core.type.TypeReference; 11 | import java.util.Collection; 12 | import lombok.NonNull; 13 | import lombok.RequiredArgsConstructor; 14 | import org.apache.curator.framework.CuratorFramework; 15 | 16 | /** Represents an implement of {@link StateRepositoryFactory} in Zookeeper. */ 17 | @RequiredArgsConstructor 18 | public final class ZookeeperRepositoryFactory implements StateRepositoryFactory { 19 | @NonNull private final CuratorFramework zkClient; 20 | 21 | @Override 22 | public ZookeeperRepository getStateRepository( 23 | String sourceName, String parition) { 24 | return new ZookeeperRepository<>( 25 | zkClient, 26 | String.format("/spinaltap/pipe/%s/state", sourceName), 27 | new TypeReference() {}); 28 | } 29 | 30 | @Override 31 | public ZookeeperRepository> getStateHistoryRepository( 32 | String sourceName, String partition) { 33 | return new ZookeeperRepository<>( 34 | zkClient, 35 | String.format("/spinaltap/pipe/%s/state_history", sourceName), 36 | new TypeReference>() {}); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /spinaltap-standalone/src/main/resources/log4j2.yaml: -------------------------------------------------------------------------------- 1 | Configuration: 2 | status: warn 3 | appenders: 4 | Console: 5 | name: STDOUT 6 | PatternLayout: 7 | Pattern: "%-5p [%d] - %t - %c: %m %n" 8 | Loggers: 9 | Root: 10 | level: info 11 | AppenderRef: 12 | ref: STDOUT 13 | -------------------------------------------------------------------------------- /spinaltap-standalone/src/main/resources/spinaltap-standalone-config-sample.yaml: -------------------------------------------------------------------------------- 1 | zk-connection-string: localhost:2181 2 | zk-namespace: spinaltap-standalone 3 | kafka-config: 4 | mysql-user: spinaltap 5 | mysql-password: spinaltap 6 | mysql-sources: 7 | - name: localhost 8 | host: localhost 9 | port: 3306 10 | tables: 11 | - test:users 12 | - test:places 13 | destination: 14 | buffer_size: 1000 --------------------------------------------------------------------------------