├── .circleci
    └── config.yml
├── .dockerignore
├── .editorconfig
├── .gitattributes
├── .github
    └── workflows
    │   └── ci.yaml
├── .gitignore
├── CHANGELOG.md
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── bin
    ├── maxwell
    ├── maxwell-benchmark
    ├── maxwell-bootstrap
    └── maxwell-docker
├── build
    ├── Gemfile
    ├── Gemfile.lock
    ├── README.md
    ├── bump
    ├── current_rev
    ├── mkchangelog
    └── release
├── config.properties.example
├── docs
    ├── CNAME
    ├── build
    ├── clean
    ├── docs
    │   ├── bootstrapping.md
    │   ├── changelog.md
    │   ├── compat.md
    │   ├── config.md
    │   ├── dataformat.md
    │   ├── deployment.md
    │   ├── embedding.md
    │   ├── encryption.md
    │   ├── filtering.md
    │   ├── high_availability.md
    │   ├── img
    │   │   ├── cyberiad_1.jpg
    │   │   ├── cyberiad_2.jpg
    │   │   └── maxwell_equals_2.png
    │   ├── index.md
    │   ├── maxwell.css
    │   ├── monitoring.md
    │   ├── producers.md
    │   ├── quickstart.md
    │   └── schemas.md
    ├── maxwell_theme
    │   ├── main.html
    │   └── toc.html
    ├── mkdocs.yml
    └── push
├── img
    └── cyberiad_1.jpg
├── kinesis-producer-library.properties.example
├── output.txt
├── pom.xml
├── raft.xml.example
└── src
    ├── example
        ├── com
        │   └── zendesk
        │   │   └── maxwell
        │   │       └── example
        │   │           ├── maxwellhealthcheckfactory
        │   │               ├── CustomMaxwellHealthCheck.java
        │   │               └── CustomMaxwellHealthCheckFactory.java
        │   │           └── producerfactory
        │   │               ├── CustomProducer.java
        │   │               └── CustomProducerFactory.java
        └── filter.js
    ├── main
        ├── antlr4
        │   ├── com
        │   │   └── zendesk
        │   │   │   └── maxwell
        │   │   │       └── schema
        │   │   │           └── ddl
        │   │   │               └── mysql.g4
        │   └── imports
        │   │   ├── column_definitions.g4
        │   │   ├── generate_tokens.rb
        │   │   ├── mysql_alter_database.g4
        │   │   ├── mysql_alter_table.g4
        │   │   ├── mysql_create_database.g4
        │   │   ├── mysql_create_table.g4
        │   │   ├── mysql_drop.g4
        │   │   ├── mysql_idents.g4
        │   │   ├── mysql_indices.g4
        │   │   ├── mysql_literal_tokens.g4
        │   │   ├── mysql_partition.g4
        │   │   ├── mysql_rename.g4
        │   │   └── mysql_view.g4
        ├── assembly
        │   └── assembly.xml
        ├── java
        │   └── com
        │   │   └── zendesk
        │   │       └── maxwell
        │   │           ├── BufferedMaxwell.java
        │   │           ├── CaseSensitivity.java
        │   │           ├── Maxwell.java
        │   │           ├── MaxwellCompatibilityError.java
        │   │           ├── MaxwellConfig.java
        │   │           ├── MaxwellContext.java
        │   │           ├── MaxwellHA.java
        │   │           ├── MaxwellMysqlConfig.java
        │   │           ├── MaxwellMysqlStatus.java
        │   │           ├── bootstrap
        │   │               ├── BootstrapController.java
        │   │               ├── BootstrapTask.java
        │   │               ├── MaxwellBootstrapUtility.java
        │   │               ├── MaxwellBootstrapUtilityConfig.java
        │   │               ├── SynchronousBootstrapper.java
        │   │               └── package-info.java
        │   │           ├── errors
        │   │               ├── DuplicateProcessException.java
        │   │               ├── ParseException.java
        │   │               ├── ProtectedAttributeNameException.java
        │   │               └── package-info.java
        │   │           ├── filtering
        │   │               ├── Filter.java
        │   │               ├── FilterColumnPattern.java
        │   │               ├── FilterParser.java
        │   │               ├── FilterPattern.java
        │   │               ├── FilterPatternType.java
        │   │               ├── FilterResult.java
        │   │               ├── InvalidFilterException.java
        │   │               └── package-info.java
        │   │           ├── monitoring
        │   │               ├── DiagnosticHealthCheck.java
        │   │               ├── DiagnosticHealthCheckModule.java
        │   │               ├── IndexListServlet.java
        │   │               ├── MaxwellConfigServlet.java
        │   │               ├── MaxwellDiagnostic.java
        │   │               ├── MaxwellDiagnosticContext.java
        │   │               ├── MaxwellDiagnosticResult.java
        │   │               ├── MaxwellHTTPServer.java
        │   │               ├── MaxwellHealthCheck.java
        │   │               ├── MaxwellHealthCheckFactory.java
        │   │               ├── MaxwellMetrics.java
        │   │               ├── Metrics.java
        │   │               ├── NoOpMetrics.java
        │   │               └── package-info.java
        │   │           ├── package-info.java
        │   │           ├── producer
        │   │               ├── AbstractAsyncProducer.java
        │   │               ├── AbstractProducer.java
        │   │               ├── AbstractProfilingProducer.java
        │   │               ├── BufferedProducer.java
        │   │               ├── EncryptionMode.java
        │   │               ├── FileProducer.java
        │   │               ├── InflightMessageList.java
        │   │               ├── KafkaProducerDiagnostic.java
        │   │               ├── MaxwellBigQueryProducer.java
        │   │               ├── MaxwellKafkaProducer.java
        │   │               ├── MaxwellKinesisProducer.java
        │   │               ├── MaxwellOutputConfig.java
        │   │               ├── MaxwellPubsubProducer.java
        │   │               ├── MaxwellRedisProducer.java
        │   │               ├── MaxwellSNSProducer.java
        │   │               ├── MaxwellSQSProducer.java
        │   │               ├── NatsProducer.java
        │   │               ├── NoneProducer.java
        │   │               ├── ProducerFactory.java
        │   │               ├── ProfilerProducer.java
        │   │               ├── RabbitmqProducer.java
        │   │               ├── StdoutProducer.java
        │   │               ├── package-info.java
        │   │               └── partitioners
        │   │               │   ├── AbstractMaxwellPartitioner.java
        │   │               │   ├── HashFunction.java
        │   │               │   ├── HashFunctionDefault.java
        │   │               │   ├── HashFunctionMurmur3.java
        │   │               │   ├── MaxwellKafkaPartitioner.java
        │   │               │   ├── MaxwellKinesisPartitioner.java
        │   │               │   ├── MaxwellSNSPartitioner.java
        │   │               │   ├── MaxwellSQSPartitioner.java
        │   │               │   ├── PartitionBy.java
        │   │               │   └── package-info.java
        │   │           ├── recovery
        │   │               ├── Recovery.java
        │   │               ├── RecoveryFilter.java
        │   │               ├── RecoveryInfo.java
        │   │               ├── RecoverySchemaStore.java
        │   │               └── package-info.java
        │   │           ├── replication
        │   │               ├── BinlogConnectorDiagnostic.java
        │   │               ├── BinlogConnectorEvent.java
        │   │               ├── BinlogConnectorEventListener.java
        │   │               ├── BinlogConnectorLivenessMonitor.java
        │   │               ├── BinlogConnectorReplicator.java
        │   │               ├── BinlogPosition.java
        │   │               ├── HeartbeatNotifier.java
        │   │               ├── MysqlVersion.java
        │   │               ├── Position.java
        │   │               ├── Replicator.java
        │   │               ├── ReplicatorNotReadyException.java
        │   │               ├── TableCache.java
        │   │               └── package-info.java
        │   │           ├── row
        │   │               ├── DataJsonGenerator.java
        │   │               ├── EncryptionContext.java
        │   │               ├── FieldNameStrategy.java
        │   │               ├── FieldNames.java
        │   │               ├── HeartbeatRowMap.java
        │   │               ├── MaxwellJson.java
        │   │               ├── RawJSONString.java
        │   │               ├── RowEncrypt.java
        │   │               ├── RowIdentity.java
        │   │               ├── RowMap.java
        │   │               ├── RowMapBuffer.java
        │   │               ├── RowMapBufferByTable.java
        │   │               ├── RowMapDeserializer.java
        │   │               └── package-info.java
        │   │           ├── schema
        │   │               ├── AbstractSchemaStore.java
        │   │               ├── BqToBqStorageSchemaConverter.java
        │   │               ├── Database.java
        │   │               ├── MysqlPositionStore.java
        │   │               ├── MysqlSavedSchema.java
        │   │               ├── MysqlSchemaCompactor.java
        │   │               ├── MysqlSchemaStore.java
        │   │               ├── PositionStoreThread.java
        │   │               ├── ReadOnlyMysqlPositionStore.java
        │   │               ├── Schema.java
        │   │               ├── SchemaCapturer.java
        │   │               ├── SchemaStore.java
        │   │               ├── SchemaStoreException.java
        │   │               ├── SchemaStoreSchema.java
        │   │               ├── Table.java
        │   │               ├── TableColumnList.java
        │   │               ├── columndef
        │   │               │   ├── BigIntColumnDef.java
        │   │               │   ├── BitColumnDef.java
        │   │               │   ├── ColumnDef.java
        │   │               │   ├── ColumnDefCastException.java
        │   │               │   ├── ColumnDefDeserializer.java
        │   │               │   ├── ColumnDefSerializer.java
        │   │               │   ├── ColumnDefWithLength.java
        │   │               │   ├── DateColumnDef.java
        │   │               │   ├── DateFormatter.java
        │   │               │   ├── DateTimeColumnDef.java
        │   │               │   ├── DecimalColumnDef.java
        │   │               │   ├── EnumColumnDef.java
        │   │               │   ├── EnumeratedColumnDef.java
        │   │               │   ├── FloatColumnDef.java
        │   │               │   ├── GeometryColumnDef.java
        │   │               │   ├── IntColumnDef.java
        │   │               │   ├── JsonColumnDef.java
        │   │               │   ├── SetColumnDef.java
        │   │               │   ├── StringColumnDef.java
        │   │               │   ├── TimeColumnDef.java
        │   │               │   ├── YearColumnDef.java
        │   │               │   └── package-info.java
        │   │               ├── ddl
        │   │               │   ├── .ruby-version
        │   │               │   ├── AddColumnMod.java
        │   │               │   ├── ChangeColumnMod.java
        │   │               │   ├── ColumnMod.java
        │   │               │   ├── ColumnPosition.java
        │   │               │   ├── DDLMap.java
        │   │               │   ├── DatabaseAlter.java
        │   │               │   ├── DatabaseCreate.java
        │   │               │   ├── DatabaseDrop.java
        │   │               │   ├── DeferredPositionUpdate.java
        │   │               │   ├── InvalidSchemaError.java
        │   │               │   ├── MaxwellSQLSyntaxError.java
        │   │               │   ├── MysqlParserListener.java
        │   │               │   ├── RemoveColumnMod.java
        │   │               │   ├── RenameColumnMod.java
        │   │               │   ├── ReparseSQLException.java
        │   │               │   ├── ResolvedDatabaseAlter.java
        │   │               │   ├── ResolvedDatabaseCreate.java
        │   │               │   ├── ResolvedDatabaseDrop.java
        │   │               │   ├── ResolvedSchemaChange.java
        │   │               │   ├── ResolvedTableAlter.java
        │   │               │   ├── ResolvedTableCreate.java
        │   │               │   ├── ResolvedTableDrop.java
        │   │               │   ├── SchemaChange.java
        │   │               │   ├── TableAlter.java
        │   │               │   ├── TableCreate.java
        │   │               │   ├── TableDrop.java
        │   │               │   └── package-info.java
        │   │               └── package-info.java
        │   │           ├── scripting
        │   │               ├── Scripting.java
        │   │               ├── WrappedDDLMap.java
        │   │               ├── WrappedHeartbeatMap.java
        │   │               ├── WrappedRowMap.java
        │   │               └── package-info.java
        │   │           └── util
        │   │               ├── AbstractConfig.java
        │   │               ├── C3P0ConnectionPool.java
        │   │               ├── ConnectionPool.java
        │   │               ├── DynamicEnum.java
        │   │               ├── ListWithDiskBuffer.java
        │   │               ├── Logging.java
        │   │               ├── MaxwellHelpFormatter.java
        │   │               ├── MaxwellOptionParser.java
        │   │               ├── MurmurHash3.java
        │   │               ├── RunLoopProcess.java
        │   │               ├── RunState.java
        │   │               ├── Sql.java
        │   │               ├── StoppableTask.java
        │   │               ├── StoppableTaskState.java
        │   │               ├── TaskManager.java
        │   │               ├── TopicInterpolator.java
        │   │               └── package-info.java
        └── resources
        │   ├── log4j2.xml
        │   └── sql
        │       ├── maxwell_schema.sql
        │       ├── maxwell_schema_bootstrap.sql
        │       └── maxwell_schema_heartbeats.sql
    └── test
        ├── java
            └── com
            │   └── zendesk
            │       └── maxwell
            │           ├── BootstrapIntegrationTest.java
            │           ├── EmbeddedMaxwellTest.java
            │           ├── JUnitNameFilterFactory.java
            │           ├── MaxwellConfigTest.java
            │           ├── MaxwellIntegrationTest.java
            │           ├── MaxwellMysqlConfigTest.java
            │           ├── MaxwellTestJSON.java
            │           ├── MaxwellTestSupport.java
            │           ├── MaxwellTestSupportCallback.java
            │           ├── MaxwellTestWithIsolatedServer.java
            │           ├── Mysql57TestSuite.java
            │           ├── Mysql57Tests.java
            │           ├── MysqlIsolatedServer.java
            │           ├── MysqlSavedSchemaTest.java
            │           ├── TestWithNameLogging.java
            │           ├── benchmark
            │               ├── BenchmarkProducer.java
            │               ├── BenchmarkProducerFactory.java
            │               └── MaxwellBenchmark.java
            │           ├── filtering
            │               └── FilterTest.java
            │           ├── monitoring
            │               ├── DiagnosticMaxwellTest.java
            │               ├── MaxwellHttpConfigIntegrationTest.java
            │               └── MaxwellMonitorIntegrationTest.java
            │           ├── producer
            │               ├── BigQueryCallbackTest.java
            │               ├── DiagnosticCallbackTest.java
            │               ├── InflightMessageListTest.java
            │               ├── KafkaCallbackTest.java
            │               ├── KinesisCallbackTest.java
            │               ├── MaxwellKafkaProducerWorkerTest.java
            │               ├── MaxwellKinesisProducerTest.java
            │               ├── MaxwellSNSProducerTest.java
            │               ├── NatsOptionMatcher.java
            │               ├── NatsProducerTest.java
            │               ├── PubsubCallbackTest.java
            │               └── partitioners
            │               │   └── MaxwellKafkaPartitionerTest.java
            │           ├── recovery
            │               └── RecoveryTest.java
            │           ├── replication
            │               ├── BinlogConnectorReplicatorTest.java
            │               ├── HeartbeatObserverTest.java
            │               ├── MysqlVersionTest.java
            │               └── TableCacheTest.java
            │           ├── row
            │               ├── RowIdentityTest.java
            │               ├── RowMapBufferTest.java
            │               ├── RowMapDeserializerTest.java
            │               └── RowMapTest.java
            │           ├── schema
            │               ├── MysqlPositionStoreTest.java
            │               ├── MysqlSchemaCompactorTest.java
            │               ├── MysqlSchemaStoreTest.java
            │               ├── PositionStoreThreadTest.java
            │               ├── SchemaCaptureTest.java
            │               ├── columndef
            │               │   └── ColumnDefTest.java
            │               └── ddl
            │               │   ├── DDLIntegrationTest.java
            │               │   ├── DDLParserTest.java
            │               │   ├── DDLResolverTest.java
            │               │   ├── DDLSerializationTest.java
            │               │   └── LowerCaseDDLIntegrationTest.java
            │           ├── scripting
            │               └── ScriptingTest.java
            │           └── util
            │               ├── StoppableTaskStateTest.java
            │               ├── TaskManagerTest.java
            │               └── TopicInterpolatorTest.java
        ├── onetimeserver
        ├── resources
            ├── config
            │   ├── env-var-config.properties
            │   └── producer-factory-config.properties
            ├── ddl
            │   ├── 5.7
            │   │   ├── alter-database
            │   │   ├── alter-event
            │   │   ├── alter-function
            │   │   ├── alter-instance
            │   │   ├── alter-procedure
            │   │   ├── alter-server
            │   │   ├── alter-table
            │   │   ├── alter-tablespace
            │   │   ├── alter-user
            │   │   ├── alter-view
            │   │   ├── analyze-table
            │   │   ├── cache-index
            │   │   ├── checksum-table
            │   │   ├── create-database
            │   │   ├── create-event
            │   │   ├── create-function-udf
            │   │   ├── create-index
            │   │   ├── create-procedure
            │   │   ├── create-server
            │   │   ├── create-table
            │   │   ├── create-tablespace
            │   │   ├── create-trigger
            │   │   ├── create-user
            │   │   ├── create-view
            │   │   ├── drop-database
            │   │   ├── drop-event
            │   │   ├── drop-function-udf
            │   │   ├── drop-index
            │   │   ├── drop-procedure
            │   │   ├── drop-server
            │   │   ├── drop-table
            │   │   ├── drop-tablespace
            │   │   ├── drop-trigger
            │   │   ├── drop-user
            │   │   ├── drop-view
            │   │   ├── grant
            │   │   ├── optimize-table
            │   │   ├── rename-table
            │   │   ├── rename-user
            │   │   ├── repair-table
            │   │   ├── truncate-table
            │   │   └── xa-statements
            │   ├── 8.0
            │   │   ├── alter-database
            │   │   ├── alter-event
            │   │   ├── alter-function
            │   │   ├── alter-instance
            │   │   ├── alter-procedure
            │   │   ├── alter-resource-group
            │   │   ├── alter-server
            │   │   ├── alter-table
            │   │   ├── alter-tablespace
            │   │   ├── alter-user
            │   │   ├── alter-view
            │   │   ├── analyze-table
            │   │   ├── cache-index
            │   │   ├── checksum-table
            │   │   ├── create-database
            │   │   ├── create-event
            │   │   ├── create-function-udf
            │   │   ├── create-index
            │   │   ├── create-procedure
            │   │   ├── create-resource-group
            │   │   ├── create-role
            │   │   ├── create-server
            │   │   ├── create-spatial-reference-system
            │   │   ├── create-table
            │   │   ├── create-tablespace
            │   │   ├── create-trigger
            │   │   ├── create-user
            │   │   ├── create-view
            │   │   ├── drop-database
            │   │   ├── drop-event
            │   │   ├── drop-function-udf
            │   │   ├── drop-index
            │   │   ├── drop-procedure
            │   │   ├── drop-resource-group
            │   │   ├── drop-role
            │   │   ├── drop-server
            │   │   ├── drop-spatial-reference-system
            │   │   ├── drop-table
            │   │   ├── drop-tablespace
            │   │   ├── drop-trigger
            │   │   ├── drop-user
            │   │   ├── drop-view
            │   │   ├── grant
            │   │   ├── optimize-table
            │   │   ├── rename-table
            │   │   ├── rename-user
            │   │   ├── repair-table
            │   │   ├── truncate-table
            │   │   └── xa-statements
            │   ├── Gemfile
            │   ├── Gemfile.lock
            │   └── get_ddl.rb
            ├── json
            │   ├── all-encrypted-user-insert.json
            │   ├── all-encrypted-user-update.json
            │   ├── encrypted-user-insert.json
            │   ├── encrypted-user-update.json
            │   ├── user-insert.json
            │   └── user-update.json
            ├── log4j2.xml
            ├── logging.properties
            ├── scripting
            │   ├── test-no-state-suppress-row.js
            │   └── test-set-state.js
            └── sql
            │   ├── ddl
            │       ├── mysql-test-errors.sql
            │       ├── mysql-test-fixed.sql
            │       ├── mysql-test-gis.sql
            │       ├── mysql-test-partition.sql
            │       └── mysql-test.sql
            │   ├── json
            │       ├── bootstrap-js-filters
            │       ├── bootstrap-json-type
            │       ├── bootstrap-multiple-row
            │       ├── bootstrap-multiple-row-with-whereclause
            │       ├── bootstrap-multiple-tables
            │       ├── bootstrap-no-pk
            │       ├── bootstrap-null-values
            │       ├── bootstrap-second-server
            │       ├── bootstrap-single-row
            │       ├── bootstrap-whitelist
            │       ├── bootstrap-with-comment
            │       ├── filter.javascript
            │       ├── filter_rich.javascript
            │       ├── test_1j
            │       ├── test_bignum
            │       ├── test_bit
            │       ├── test_blob
            │       ├── test_case_insensitive
            │       ├── test_charsets
            │       ├── test_column_case
            │       ├── test_create_like
            │       ├── test_create_select
            │       ├── test_enum
            │       ├── test_gis
            │       ├── test_invalid_time
            │       ├── test_javascript_filters
            │       ├── test_javascript_filters_rich_values
            │       ├── test_json
            │       ├── test_latin1
            │       ├── test_minimal
            │       ├── test_row_query_log_is_on
            │       ├── test_set
            │       ├── test_time
            │       ├── test_ucs2
            │       └── test_zero_created_at
            │   ├── rows
            │       └── rows.sql
            │   ├── schema
            │       ├── enum.sql
            │       ├── mediumint.sql
            │       ├── minimal.sql
            │       ├── sharded.sql
            │       ├── sharded_55.sql
            │       └── unsigned.sql
            │   └── serialization
            │       ├── alter_table
            │       ├── alter_table_utf8mb3
            │       ├── create_database
            │       ├── create_table
            │       ├── create_table_55
            │       ├── create_table_utf8mb3
            │       └── drop_table
        └── travis.sh


/.dockerignore:
--------------------------------------------------------------------------------
1 | .git
2 | target
3 | 


--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | [*.java]
2 | charset=utf-8
3 | end_of_line=lf
4 | insert_final_newline=true
5 | indent_style=tab
6 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.min.js binary
2 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yaml:
--------------------------------------------------------------------------------
 1 | name: CI to Zendesk Dockerhub
 2 | 
 3 | on:
 4 |   push:
 5 |     tags:
 6 |       - v*.*.*
 7 | 
 8 | jobs:
 9 |   build:
10 |     runs-on: ubuntu-latest
11 |     steps:
12 |       - uses: zendesk/checkout@v2
13 |         with:
14 |           fetch-depth: '1'
15 |       - name: Set up QEMU
16 |         run: |-
17 |           QEMU_IMAGE=tonistiigi/binfmt:latest
18 |           docker pull $QEMU_IMAGE
19 |           docker image inspect $QEMU_IMAGE
20 |           docker run --rm --privileged $QEMU_IMAGE --install all
21 |       - name: Set up Docker Buildx
22 |         id: buildx
23 |         uses: zendesk/setup-buildx-action@v1.6.0
24 |         with:
25 |           install: true
26 |       - name: Docker Build and push
27 |         run: |-
28 |           set -eu -o pipefail
29 |           echo ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | docker login --username=${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin
30 |           set -x
31 |           current_tag=zendesk/maxwell:"${GITHUB_REF##refs/tags/}"
32 |           latest_tag=zendesk/maxwell:latest
33 |           docker buildx build --platform=linux/arm64,linux/amd64 --file=Dockerfile --push --tag="$current_tag" --tag="$latest_tag" .
34 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | data/*.yaml
 2 | config/*.yml
 3 | /target
 4 | /config.properties
 5 | /maxwell.position
 6 | /.idea
 7 | /*.iml
 8 | .*.sw*
 9 | .DS_Store
10 | /test.log
11 | .sw?
12 | /config
13 | .classpath
14 | .project
15 | .settings/
16 | .vscode/
17 | .factorypath
18 | .sts4-cache/
19 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM maven:3.9.9-eclipse-temurin-23 AS builder
 2 | ENV MAXWELL_VERSION=1.43.2 KAFKA_VERSION=1.0.0
 3 | 
 4 | 
 5 | RUN apt-get update \
 6 |     && apt-get -y upgrade \
 7 |     && apt-get install -y make
 8 | 
 9 | # prime so we can have a cached image of the maven deps
10 | COPY pom.xml /tmp
11 | RUN cd /tmp && mvn dependency:resolve
12 | 
13 | COPY . /workspace
14 | RUN cd /workspace \
15 |     && KAFKA_VERSION=$KAFKA_VERSION make package MAXWELL_VERSION=$MAXWELL_VERSION \
16 |     && mkdir /app \
17 |     && mv /workspace/target/maxwell-$MAXWELL_VERSION/maxwell-$MAXWELL_VERSION/* /app/ \
18 |     && apt-get clean \
19 |     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* /workspace/ /root/.m2/ \
20 |     && echo "$MAXWELL_VERSION" > /REVISION
21 | 
22 | # Build clean image with non-root priveledge
23 | FROM openjdk:23-jdk-slim
24 | 
25 | RUN apt-get update \
26 |     && apt-get -y upgrade
27 | 
28 | # Add curl, required to add an healthcheck as a docker command
29 | RUN apt-get -y install --no-install-recommends curl jq
30 | 
31 | COPY --from=builder /app /app
32 | COPY --from=builder /REVISION /REVISION
33 | 
34 | WORKDIR /app
35 | 
36 | RUN useradd -u 1000 maxwell -d /app
37 | RUN chown 1000:1000 /app
38 | 
39 | USER 1000
40 | 
41 | CMD [ "/bin/bash", "-c", "bin/maxwell-docker" ]
42 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 |  Copyright 2025 Zendesk
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |    http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | KAFKA_VERSION ?= 2.7.0
 2 | KAFKA_PROFILE = kafka-${KAFKA_VERSION}
 3 | export JAVA_TOOL_OPTIONS = -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
 4 | 
 5 | all: compile
 6 | 
 7 | test:
 8 | 	mvn -B test -P ${KAFKA_PROFILE}
 9 | 
10 | compile:
11 | 	mvn -B compile -P ${KAFKA_PROFILE}
12 | 
13 | clean:
14 | 	mvn -B clean
15 | 
16 | depclean: clean
17 | 	rm -f $(CLASSPATH)
18 | 
19 | package: depclean kafka-0.8.2.2 kafka-0.9.0.1 kafka-0.10.0.1 kafka-0.10.2.1 kafka-0.11.0.1 kafka-1.0.0 kafka-2.7.0 kafka-3.4.0
20 | 	@# TODO: this is inefficient, we really just want to copy the jars...
21 | 	mvn package -DskipTests=true
22 | 
23 | kafka-%:
24 | 	mvn compile -P kafka-$(*)
25 | 
26 | FORCE:
27 | docs: FORCE
28 | 	mvn javadoc:javadoc
29 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | This is __Maxwell's daemon__, a [change data capture](https://www.confluent.io/blog/how-change-data-capture-works-patterns-solutions-implementation/) application 
 2 | that reads MySQL binlogs and writes data changes as JSON to Kafka, Kinesis, and other streaming platforms.
 3 | 
 4 | 
 5 | 
 6 | [↓ Download](https://github.com/zendesk/maxwell/releases/download/v1.43.2/maxwell-1.43.2.tar.gz) \|
 7 | [⚝ Source / Community](https://github.com/zendesk/maxwell) \|
 8 | [☝ Getting Started](/quickstart) \|
 9 | [☷ Reference](/config)
10 | 
11 | __What's it for?__
12 | 
13 | - ETL of all sorts
14 | - maintaining an audit log of all changes to your database
15 | - cache building/expiring
16 | - search indexing 
17 | - inter-service communication
18 | 
19 | 
20 | __It goes like this:__
21 | 
22 | ```
23 |   mysql> update `test`.`maxwell` set mycol = 55, daemon = 'Stanislaw Lem';
24 |   maxwell -> kafka: 
25 |   {
26 |     "database": "test",
27 |     "table": "maxwell",
28 |     "type": "update",
29 |     "ts": 1449786310,
30 |     "data": { "id":1, "daemon": "Stanislaw Lem", "mycol": 55 },
31 |     "old": { "mycol":, 23, "daemon": "what once was" }
32 |   }
33 | ```
34 | 


--------------------------------------------------------------------------------
/bin/maxwell-bootstrap:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | base_dir="$(dirname "$0")/.."
 6 | lib_dir="$base_dir/lib"
 7 | lib_dir_development="$base_dir/target/lib"
 8 | 
 9 | if [ ! -e "$lib_dir" -a -e "$lib_dir_development" ]; then
10 | 	lib_dir="$lib_dir_development"
11 | 	CLASSPATH="$CLASSPATH:$base_dir/target/classes"
12 | fi
13 | 
14 | CLASSPATH="$CLASSPATH:$lib_dir/*"
15 | 
16 | if [ -z "$JAVA_HOME" ]; then
17 |   JAVA="java"
18 | else
19 |   JAVA="$JAVA_HOME/bin/java"
20 | fi
21 | 
22 | export LANG="en_US.UTF-8"
23 | 
24 | exec $JAVA -Dlog4j.shutdownCallbackRegistry=com.djdch.log4j.StaticShutdownCallbackRegistry -cp $CLASSPATH com.zendesk.maxwell.bootstrap.MaxwellBootstrapUtility "$@"
25 | 


--------------------------------------------------------------------------------
/bin/maxwell-docker:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | MAXWELL_PRODUCER=${MAXWELL_PRODUCER:-kafka}
 4 | 
 5 | if [ "$MAXWELL_PRODUCER" == "kafka" ]
 6 | then
 7 |   if [ -z "$KAFKA_BROKERS" ]
 8 |   then
 9 |     KAFKA_BROKERS="$KAFKA_HOST:$KAFKA_PORT"
10 |   fi
11 |   MAXWELL_OPTIONS="$MAXWELL_OPTIONS --kafka.bootstrap.servers=$KAFKA_BROKERS"
12 | fi
13 | 
14 | exec `dirname $0`/maxwell  --user=$MYSQL_USERNAME --password=$MYSQL_PASSWORD --host=$MYSQL_HOST --producer=$MAXWELL_PRODUCER $MAXWELL_OPTIONS
15 | 


--------------------------------------------------------------------------------
/build/Gemfile:
--------------------------------------------------------------------------------
1 | 
2 | source 'https://rubygems.org'
3 | gem 'octokit', '>= 4.6.0'
4 | gem 'netrc'
5 | gem 'mime-types'
6 | 
7 | 


--------------------------------------------------------------------------------
/build/Gemfile.lock:
--------------------------------------------------------------------------------
 1 | GEM
 2 |   remote: https://rubygems.org/
 3 |   specs:
 4 |     addressable (2.8.0)
 5 |       public_suffix (>= 2.0.2, < 5.0)
 6 |     faraday (1.5.0)
 7 |       faraday-em_http (~> 1.0)
 8 |       faraday-em_synchrony (~> 1.0)
 9 |       faraday-excon (~> 1.1)
10 |       faraday-httpclient (~> 1.0.1)
11 |       faraday-net_http (~> 1.0)
12 |       faraday-net_http_persistent (~> 1.1)
13 |       faraday-patron (~> 1.0)
14 |       multipart-post (>= 1.2, < 3)
15 |       ruby2_keywords (>= 0.0.4)
16 |     faraday-em_http (1.0.0)
17 |     faraday-em_synchrony (1.0.0)
18 |     faraday-excon (1.1.0)
19 |     faraday-httpclient (1.0.1)
20 |     faraday-net_http (1.0.1)
21 |     faraday-net_http_persistent (1.1.0)
22 |     faraday-patron (1.0.0)
23 |     mime-types (3.1)
24 |       mime-types-data (~> 3.2015)
25 |     mime-types-data (3.2016.0521)
26 |     multipart-post (2.1.1)
27 |     netrc (0.11.0)
28 |     octokit (4.21.0)
29 |       faraday (>= 0.9)
30 |       sawyer (~> 0.8.0, >= 0.5.3)
31 |     public_suffix (4.0.6)
32 |     ruby2_keywords (0.0.4)
33 |     sawyer (0.8.2)
34 |       addressable (>= 2.3.5)
35 |       faraday (> 0.8, < 2.0)
36 | 
37 | PLATFORMS
38 |   ruby
39 | 
40 | DEPENDENCIES
41 |   mime-types
42 |   netrc
43 |   octokit (>= 4.6.0)
44 | 
45 | BUNDLED WITH
46 |    1.17.3
47 | 


--------------------------------------------------------------------------------
/build/bump:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | if [ "$1" == "" ]
 4 | then
 5 |   echo "Usage: $0 VERSION" && exit 0
 6 | fi
 7 | 
 8 | DIR=`dirname $0`
 9 | cd $DIR/..
10 | 
11 | CURRENT_REV="$2"
12 | if [ -z "$CURRENT_REV" ] ; then
13 |   CURRENT_REV=`build/current_rev`
14 | 
15 |   if [ "$CURRENT_REV" == "" ]
16 |   then
17 |     echo "Couldn't get CURRENT_REV" && exit 1
18 |   fi
19 | fi
20 | 
21 | git grep -lF $CURRENT_REV | grep -v CHANGELOG.md | xargs perl -i -npe "s|$CURRENT_REV|$1|g"
22 | 


--------------------------------------------------------------------------------
/build/current_rev:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | POM=`dirname $0`/../pom.xml
4 | grep '<version>' $POM | head -n1 | perl -npe 'chomp; s%.*<version>(.*)\s*</version>%\1%'
5 | 


--------------------------------------------------------------------------------
/build/mkchangelog:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env ruby
 2 | 
 3 | require 'bundler/setup'
 4 | require 'octokit'
 5 | 
 6 | client = Octokit::Client.new(:netrc => true)
 7 | 
 8 | puts "# Maxwell changelog"
 9 | puts
10 | 
11 | page = 1
12 | releases = client.releases('zendesk/maxwell', page: page)
13 | 
14 | while releases.any?
15 |   releases.each do |r|
16 |     prerelease = !!(r[:tag_name] =~ /-\w+$/)
17 |     next if prerelease && r[:body].strip.empty?
18 |     out = <<-EOL
19 | ### [%{tag_name}](%{html_url})
20 | 
21 | %{body}
22 | 
23 | _Released %{date}_
24 |     EOL
25 | 
26 |     r[:body].gsub!(/\r\n/, "\n")
27 |     r[:date] = r[:created_at].to_s.split(' ').first
28 |     puts out.strip % r
29 |     puts
30 |   end
31 | 
32 |   page = page + 1
33 |   releases = client.releases('zendesk/maxwell', page: page)
34 | end
35 | 
36 | 
37 | 


--------------------------------------------------------------------------------
/docs/CNAME:
--------------------------------------------------------------------------------
1 | maxwells-daemon.io
2 | 


--------------------------------------------------------------------------------
/docs/build:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | set -eux
4 | cd $(dirname $0)
5 | ./clean
6 | git worktree add site origin/gh-pages
7 | mkdocs build --clean && cp CNAME site
8 | 
9 | 


--------------------------------------------------------------------------------
/docs/clean:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | set -eux
4 | cd $(dirname $0)
5 | rm -rf site
6 | git worktree prune
7 | 
8 | 


--------------------------------------------------------------------------------
/docs/docs/changelog.md:
--------------------------------------------------------------------------------
1 | ../../CHANGELOG.md


--------------------------------------------------------------------------------
/docs/docs/embedding.md:
--------------------------------------------------------------------------------
 1 | # Embedding Maxwell
 2 | ***
 3 | Maxwell typically runs as a command-line program. However, for advanced uses it
 4 | is possible to run maxwell from any JVM-based language.
 5 | 
 6 | Some fairly incomplete API documentation is available here:<br/>
 7 | [https://maxwells-daemon.io/apidocs](https://maxwells-daemon.io/apidocs)
 8 | 
 9 | # Compatibility caveat
10 | ***
11 | Maxwell makes every attempt to remain backwards compatible. However this
12 | only applies to the command-line usage - Maxwell's Java API may change without
13 | notice.
14 | 
15 | However (and unless otherwise indicated) breaking API changes will result in
16 | a type error - i.e. if your code still compiles, then the API has not changed.
17 | 


--------------------------------------------------------------------------------
/docs/docs/encryption.md:
--------------------------------------------------------------------------------
 1 | # Using encryption
 2 | ***
 3 | When encryption is enabled, maxwell will encrypt messages using a AES/CBC/PKCS5PADDING cipher with your own encryption key.
 4 | Values are first encrypted and then base64 encoded, an initialization vector is randomly generated and put into the final message
 5 | 
 6 | # Decryption
 7 | ***
 8 | To decrypt your data you must first decode the string from base64 and then apply the cipher to decrypt. A sample implementation is provided in RowEncrypt.decrypt().
 9 | 
10 | # Examples
11 | ***
12 | 
13 | insert into minimal set account_id =1, text_field='hello'
14 | 
15 | `encrypt=none` (unencrypted):
16 | 
17 | ```
18 | {"database":"shard_1","table":"minimal","type":"insert","ts":1490115785,"xid":153,"commit":true,"data":{"id":1,"account_id":1,"text_field":"hello"}}
19 | ```
20 | 
21 | `encrypt=data`:
22 | 
23 | ```
24 | {"database":"shard_1","table":"minimal","type":"insert","ts":1504585129,"xid":161,"commit":true,"encrypted":{"iv":"lqiXoTdz6jed3XgJPpa7EQ==","bytes":"1soc4leskiIm6yuT2D49VA3AYVKCvN+0wh+8d1iwSZETK7N2pG4HDbqnVpJUUCOaKjpcPlP7Sc7Z3SPhGD5JeA=="}}
25 | ```
26 | 
27 | `encrypt=all`:
28 | 
29 | ```
30 | {"encrypted":{"bytes":"iZssjWfzS0NlqIj82ddpvoQeKSx4D3GIPSCgjdkpgQlCWzN2p3VVZOn3Oj1x4w+a6dVhoFmllWxBK6aAkdVK9t6Vt1+um6lWwSeXNQIL/RbknW5Q8I9emm5bC1Dd1LftBuX/1Uw0wjbsq8Qt3HErvmmiIMe2S27EIWshvBnmw9MibryjLD0brvIbFFxwDuSQuVA4OFyV9TN32N/ZXiBwIA==","iv":"XXs6AePsXJWAAIrKyLlR0g=="}}
31 | ```
32 | 


--------------------------------------------------------------------------------
/docs/docs/img/cyberiad_1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zendesk/maxwell/ae1c2e926f9ad85dd342fc2afd998436b4ed5db7/docs/docs/img/cyberiad_1.jpg


--------------------------------------------------------------------------------
/docs/docs/img/cyberiad_2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zendesk/maxwell/ae1c2e926f9ad85dd342fc2afd998436b4ed5db7/docs/docs/img/cyberiad_2.jpg


--------------------------------------------------------------------------------
/docs/docs/img/maxwell_equals_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zendesk/maxwell/ae1c2e926f9ad85dd342fc2afd998436b4ed5db7/docs/docs/img/maxwell_equals_2.png


--------------------------------------------------------------------------------
/docs/docs/index.md:
--------------------------------------------------------------------------------
1 | ../../README.md


--------------------------------------------------------------------------------
/docs/docs/maxwell.css:
--------------------------------------------------------------------------------
 1 | #Reference table {
 2 |   width: 100%;
 3 |   font-size: 80%;
 4 | }
 5 | 
 6 | #Reference table td:nth-child(1) {
 7 |   font-family: monospace;
 8 |   font-size: 120%;
 9 | }
10 | #Reference table td:nth-child(2) {
11 | }
12 | #Reference table td:nth-child(3) {
13 | }
14 | #Reference table td:nth-child(4) {
15 | }
16 | 
17 | #Monitoring table {
18 |   width: 100%;
19 |   font-size: 80%;
20 | }
21 | 
22 | #Monitoring table td:nth-child(1) {
23 |   width: 25%;
24 | }
25 | 
26 | #Changelog h3 {
27 |   font-size: 1.25rem;
28 | }
29 | 
30 | blockquote {
31 |   background: #f9f9f9;
32 |   border-left: 10px solid #ccc;
33 |   margin: 1.5em 10px;
34 |   padding: 0.5em 10px;
35 | }
36 | 
37 | h1:not(:first-child) {
38 |   margin-top: 60px;
39 | }
40 | 
41 | .bs-sidebar .nav > li > a {
42 |   padding: 3px 20px;
43 | }
44 | 


--------------------------------------------------------------------------------
/docs/maxwell_theme/main.html:
--------------------------------------------------------------------------------
 1 | {% extends "base.html" %}
 2 | {%- block content %}
 3 | <div class="col-md-4">{% include "toc.html" %}</div>
 4 | <div class="col-md-8" role="main">
 5 |     <div id="{{ page.title }}">
 6 |         {% include "content.html" %}
 7 |     </div>
 8 | </div>
 9 | {%- endblock %}
10 | 


--------------------------------------------------------------------------------
/docs/maxwell_theme/toc.html:
--------------------------------------------------------------------------------
 1 | {%- if page.toc.items.__len__() > 0 %}
 2 | <div class="navbar-light navbar-expand-md bs-sidebar hidden-print affix" role="complementary">
 3 |     <div class="navbar-header">
 4 |         <button type="button" class="navbar-toggler collapsed" data-toggle="collapse" data-target="#toc-collapse" title="Table of Contents">
 5 |             <span class="fa fa-angle-down"></span>
 6 |         </button>
 7 |     </div>
 8 | 
 9 |     {% macro toc_item(item) %}
10 |         {%- if item.level <= config.theme.navigation_depth %}
11 |             <li class="nav-item" data-level="{{ item.level }}"><a href="{{ item.url }}" class="nav-link">{{ item.title }}</a>
12 |               <ul class="nav flex-column">
13 |                 {%- for child in item.children %}
14 |                     {{- toc_item(child) }}
15 |                 {%- endfor %}
16 |               </ul>
17 |             </li>
18 |         {%- endif %}
19 |     {%- endmacro %}
20 |     <div id="toc-collapse" class="navbar-collapse collapse card bg-secondary">
21 |         <ul class="nav flex-column">
22 |         {%- for item in page.toc %}
23 |             {{ toc_item(item) }}
24 |         {%- endfor %}
25 |         </ul>
26 |     </div>
27 | </div>
28 | {% else %}
29 | <div>
30 |     <img alt='The Daemon, maybe' src='/img/cyberiad_1.jpg' id='maxwell-daemon-image' style="width: 100%">
31 | </div>
32 | {% endif %}
33 | 


--------------------------------------------------------------------------------
/docs/mkdocs.yml:
--------------------------------------------------------------------------------
 1 | site_name: Maxwell's Daemon
 2 | theme:
 3 |   name: mkdocs
 4 |   custom_dir: maxwell_theme
 5 | extra_css:
 6 |   - maxwell.css
 7 | nav:
 8 |   - 'Quick Start': 'quickstart.md'
 9 |   - 'Documentation':
10 |     - 'Reference': 'config.md'
11 |     - 'Deployment': 'deployment.md'
12 |     - 'Producers': 'producers.md'
13 |     - 'Filtering': 'filtering.md'
14 |     - 'Bootstrapping': 'bootstrapping.md'
15 |     - 'Data Format': 'dataformat.md'
16 |     - 'Encryption': 'encryption.md'
17 |     - 'Monitoring': 'monitoring.md'
18 |     - 'High Availability': 'high_availability.md'
19 |     - 'Embedding': 'embedding.md'
20 |     - 'Internals': 'schemas.md'
21 |     - 'Compatibility': 'compat.md'
22 |   - 'Changelog': 'changelog.md'
23 | 


--------------------------------------------------------------------------------
/docs/push:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -eux
 4 | make docs
 5 | 
 6 | cd $(dirname $0)
 7 | ./build
 8 | cp -a ../target/site/apidocs site/
 9 | (cd site && git add . && git commit -m 'update docs' && git push origin HEAD:gh-pages)
10 | ./clean
11 | 


--------------------------------------------------------------------------------
/img/cyberiad_1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zendesk/maxwell/ae1c2e926f9ad85dd342fc2afd998436b4ed5db7/img/cyberiad_1.jpg


--------------------------------------------------------------------------------
/output.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zendesk/maxwell/ae1c2e926f9ad85dd342fc2afd998436b4ed5db7/output.txt


--------------------------------------------------------------------------------
/raft.xml.example:
--------------------------------------------------------------------------------
 1 | <?xml version='1.0' encoding='utf-8'?>
 2 | <config xmlns="urn:org:jgroups"
 3 |         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |         xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/jgroups.xsd">
 5 |     <UDP mcast_addr="228.8.8.8" mcast_port="${jgroups.udp.mcast_port:45588}"/>
 6 |     <PING />
 7 |     <MERGE3 />
 8 |     <FD_SOCK/>
 9 |     <FD_ALL/>
10 |     <VERIFY_SUSPECT timeout="1500"/>
11 |     <pbcast.NAKACK2 xmit_interval="500"/>
12 |     <UNICAST3 xmit_interval="500"/>
13 |     <pbcast.STABLE desired_avg_gossip="50000" max_bytes="4M"/>
14 |     <raft.NO_DUPES/>
15 |     <pbcast.GMS print_local_addr="true" join_timeout="2000"/>
16 |     <UFC max_credits="2M" min_threshold="0.4"/>
17 |     <MFC max_credits="2M" min_threshold="0.4"/>
18 |     <FRAG2 frag_size="60K"/>
19 |     <raft.ELECTION election_min_interval="500" election_max_interval="1000" heartbeat_interval="250"/>
20 |     <raft.RAFT members="A,B,C" raft_id="${raft_id:undefined}"/>
21 |     <raft.REDIRECT/>
22 | </config>
23 | 


--------------------------------------------------------------------------------
/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheck.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.example.maxwellhealthcheckfactory;
 2 | 
 3 | import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
 4 | import com.zendesk.maxwell.producer.AbstractProducer;
 5 | 
 6 | public class CustomMaxwellHealthCheck extends MaxwellHealthCheck {
 7 | 	public CustomMaxwellHealthCheck(AbstractProducer producer) {
 8 | 		super(producer);
 9 | 	}
10 | 
11 | 	@Override
12 | 	protected Result check() throws Exception {
13 | 		return Result.unhealthy("I am always unhealthy");
14 | 	}
15 | }
16 | 


--------------------------------------------------------------------------------
/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheckFactory.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.example.maxwellhealthcheckfactory;
 2 | 
 3 | import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
 4 | import com.zendesk.maxwell.monitoring.MaxwellHealthCheckFactory;
 5 | import com.zendesk.maxwell.producer.AbstractProducer;
 6 | 
 7 | public class CustomMaxwellHealthCheckFactory implements MaxwellHealthCheckFactory
 8 | {
 9 | 	@Override
10 | 	public MaxwellHealthCheck createHealthCheck(AbstractProducer producer)
11 | 	{
12 | 		return new CustomMaxwellHealthCheck(producer);
13 | 	}
14 | }
15 | 


--------------------------------------------------------------------------------
/src/example/com/zendesk/maxwell/example/producerfactory/CustomProducerFactory.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.example.producerfactory;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.producer.AbstractProducer;
 5 | import com.zendesk.maxwell.producer.ProducerFactory;
 6 | 
 7 | /**
 8 |  * Custom {@link ProducerFactory} example that creates a new {@link CustomProducer}. To register your custom producer, 
 9 |  * set the {@code custom_producer.factory} property to your custom producer factory's fully qualified class name.
10 |  */
11 | public class CustomProducerFactory implements ProducerFactory
12 | {
13 | 	@Override
14 | 	public AbstractProducer createProducer(MaxwellContext context)
15 | 	{
16 | 		return new CustomProducer(context);
17 | 	}
18 | }
19 | 


--------------------------------------------------------------------------------
/src/main/antlr4/com/zendesk/maxwell/schema/ddl/mysql.g4:
--------------------------------------------------------------------------------
 1 | grammar mysql;
 2 | 
 3 | import mysql_literal_tokens,
 4 |        mysql_idents,
 5 |        mysql_alter_table,
 6 |        mysql_alter_database,
 7 |        mysql_create_database,
 8 |        mysql_create_table,
 9 |        mysql_drop,
10 |        mysql_rename,
11 |        mysql_view;
12 | 
13 | parse: statement?
14 |        EOF;
15 | 
16 | statement:
17 |     alter_table
18 |   | alter_view
19 |   | alter_database
20 |   | create_database
21 |   | create_table
22 |   | create_view
23 |   | drop_database
24 |   | drop_table
25 |   | drop_view
26 |   | rename_table
27 |   | BEGIN
28 |   ;
29 | 


--------------------------------------------------------------------------------
/src/main/antlr4/imports/mysql_alter_database.g4:
--------------------------------------------------------------------------------
 1 | grammar mysql_alter_database;
 2 | 
 3 | import mysql_literal_tokens, mysql_idents;
 4 | 
 5 | alter_database: ALTER (DATABASE | SCHEMA) name? alter_database_definition;
 6 | alter_database_definition:
 7 |     (default_character_set | default_collation)+
 8 |   | UPGRADE DATA DIRECTORY NAME
 9 |   | alter_encryption;
10 | 
11 | alter_encryption: ENCRYPTION '='? string_literal;
12 | 


--------------------------------------------------------------------------------
/src/main/antlr4/imports/mysql_create_database.g4:
--------------------------------------------------------------------------------
 1 | grammar mysql_create_database;
 2 | 
 3 | import mysql_literal_tokens, mysql_idents;
 4 | 
 5 | create_option:
 6 |   default_character_set
 7 |   | default_collation
 8 |   | DEFAULT ENCRYPTION '=' string_literal;
 9 | 
10 | create_database:
11 |   CREATE (DATABASE | SCHEMA) if_not_exists? name create_option*;
12 | 
13 | 


--------------------------------------------------------------------------------
/src/main/antlr4/imports/mysql_drop.g4:
--------------------------------------------------------------------------------
1 | grammar mysql_drop;
2 | import mysql_literal_tokens, mysql_idents;
3 | 
4 | drop_database: DROP (DATABASE | SCHEMA) if_exists? name;
5 | drop_table: DROP TEMPORARY? TABLE if_exists? table_name (',' table_name)* drop_table_options*;
6 | drop_table_options: (RESTRICT | CASCADE);
7 | 
8 | if_exists: IF EXISTS;
9 | 


--------------------------------------------------------------------------------
/src/main/antlr4/imports/mysql_rename.g4:
--------------------------------------------------------------------------------
1 | grammar mysql_rename;
2 | import mysql_literal_tokens, mysql_idents;
3 | 
4 | rename_table: RENAME (TABLE | TABLES) rename_table_spec (',' rename_table_spec)*;
5 | rename_table_spec: table_name TO table_name;
6 | 
7 | 


--------------------------------------------------------------------------------
/src/main/antlr4/imports/mysql_view.g4:
--------------------------------------------------------------------------------
 1 | grammar mysql_view;
 2 | 
 3 | import mysql_literal_tokens, mysql_idents;
 4 | 
 5 | /*
 6 |   This in an intentionally incomplete grammar for parsing VIEW statements.
 7 |   It's designed to parse up to the (SELECT *), as that cruft is too tricky to
 8 |   capture with a regular-expression based blacklist.
 9 | */
10 | 
11 | alter_view:
12 |   ALTER view_options* VIEW name;
13 | 
14 | create_view:
15 |   CREATE (OR REPLACE)? view_options* VIEW name;
16 | 
17 | drop_view:
18 |   DROP VIEW (IF EXISTS)? name (',' name)* (RESTRICT | CASCADE)?;
19 | 
20 | view_options:
21 |     ALGORITHM '=' (UNDEFINED | MERGE | TEMPTABLE)
22 |   | DEFINER '=' (user | CURRENT_USER)
23 |   | SQL SECURITY ( DEFINER | INVOKER );
24 | 
25 | 
26 | 
27 | 
28 | 
29 | 
30 | 


--------------------------------------------------------------------------------
/src/main/assembly/assembly.xml:
--------------------------------------------------------------------------------
 1 | <assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
 2 |   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 3 |   xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
 4 |   <id>maxwell</id>
 5 |   <formats>
 6 |     <format>tar.gz</format>
 7 |     <format>dir</format>
 8 |   </formats>
 9 |   <fileSets>
10 |     <fileSet>
11 |       <directory>${project.basedir}</directory>
12 |       <includes>
13 |         <include>bin/*</include>
14 |         <include>config.properties.example</include>
15 |         <include>kinesis-producer-library.properties.example</include>
16 |         <include>README*</include>
17 |         <include>LICENSE*</include>
18 |         <include>NOTICE*</include>
19 |       </includes>
20 |     </fileSet>
21 |     <fileSet>
22 |       <directory>${project.build.directory}/lib</directory>
23 |       <outputDirectory>lib</outputDirectory>
24 |     </fileSet>
25 |     <fileSet>
26 |       <directory>${project.build.directory}</directory>
27 |       <outputDirectory>lib</outputDirectory>
28 |       <includes>
29 |         <include>maxwell-${project.version}.jar</include>
30 |       </includes>
31 |     </fileSet>
32 |   </fileSets>
33 |   <files>
34 |     <file>
35 |       <source>src/main/resources/log4j2.xml</source>
36 |     </file>
37 |     <file>
38 |       <source>docs/docs/config.md</source>
39 |     </file>
40 |     <file>
41 |       <source>docs/docs/quickstart.md</source>
42 |     </file>
43 |   </files>
44 | </assembly>
45 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/BufferedMaxwell.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | import com.zendesk.maxwell.producer.BufferedProducer;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | 
 6 | import java.io.IOException;
 7 | import java.net.URISyntaxException;
 8 | import java.sql.SQLException;
 9 | import java.util.concurrent.TimeUnit;
10 | 
11 | /**
12 |  * A subclass of {@link Maxwell} that buffers rows in-memory for consumption by the caller
13 |  */
14 | public class BufferedMaxwell extends Maxwell {
15 | 	/**
16 | 	 * Initializer for a buffered Maxwell instance.  Sets up buffered producer.
17 | 	 * @param config Maxwell configuration
18 | 	 * @throws SQLException if we have db issues
19 | 	 * @throws URISyntaxException if we can't build a database URI
20 | 	 */
21 | 	public BufferedMaxwell(MaxwellConfig config) throws SQLException, URISyntaxException {
22 | 		super(config);
23 | 		config.producerType = "buffer";
24 | 	}
25 | 
26 | 	/**
27 | 	 * Poll for a RowMap to be producer by the maxwell instance.
28 | 	 * @param ms poll time to wait in milliseconds before timing out
29 | 	 * @return RowMap
30 | 	 * @throws IOException if we have issues building a producer
31 | 	 * @throws InterruptedException if we are interrupted
32 | 	 */
33 | 	public RowMap poll(long ms) throws IOException, InterruptedException {
34 | 		BufferedProducer p = (BufferedProducer) this.context.getProducer();
35 | 		return p.poll(ms, TimeUnit.MILLISECONDS);
36 | 	}
37 | 
38 | 	public MaxwellContext getContext() {
39 | 		return context;
40 | 	}
41 | }
42 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/CaseSensitivity.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | /**
 4 |  * Describes the 3 difference case sensitivity settings on a Mysql server.
 5 |  * Case sensitivity rules apply to databases and tables, not columns.
 6 |  */
 7 | public enum CaseSensitivity { CASE_SENSITIVE, CONVERT_TO_LOWER, CONVERT_ON_COMPARE };
 8 | 
 9 | 
10 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/MaxwellCompatibilityError.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | /**
 4 |  * Thrown when Maxwell can't operate with the mysql server configured as it is.
 5 |  */
 6 | public class MaxwellCompatibilityError extends Exception {
 7 | 	public MaxwellCompatibilityError(String message) {
 8 | 		super(message);
 9 | 	}
10 | 	private static final long serialVersionUID = 1L;
11 | }
12 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/bootstrap/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * bootstrapping outputs the current dataset into a stream.
3 |  */
4 | 
5 | package com.zendesk.maxwell.bootstrap;
6 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/errors/DuplicateProcessException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.errors;
2 | 
3 | public class DuplicateProcessException extends Exception {
4 |         public DuplicateProcessException (String message) { super(message); }
5 |         private static final long serialVersionUID = 1L;
6 | }
7 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/errors/ParseException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.errors;
2 | 
3 | public class ParseException extends RuntimeException {
4 | 	public ParseException (String message) {
5 | 		super(message);
6 | 	}
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/errors/ProtectedAttributeNameException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.errors;
2 | 
3 | public class ProtectedAttributeNameException extends RuntimeException {
4 | 	public ProtectedAttributeNameException(String message) {
5 | 		super(message);
6 | 	}
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/errors/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * exceptions thrown by maxwell.
3 |  */
4 | 
5 | package com.zendesk.maxwell.errors;
6 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/filtering/FilterPatternType.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.filtering;
2 | 
3 | public enum FilterPatternType {
4 | 	INCLUDE, EXCLUDE, BLACKLIST
5 | }
6 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/filtering/FilterResult.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.filtering;
2 | 
3 | class FilterResult {
4 | 	public boolean include = true;
5 | }
6 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/filtering/InvalidFilterException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.filtering;
2 | 
3 | public class InvalidFilterException extends Exception  {
4 | 	public InvalidFilterException(String message) {
5 | 		super(message);
6 | 	}
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/filtering/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * filtering allows users to choose what data to output in a stream
3 |  */
4 | package com.zendesk.maxwell.filtering;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/IndexListServlet.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | 
 4 | import org.apache.commons.lang3.tuple.Pair;
 5 | 
 6 | import javax.servlet.ServletException;
 7 | import javax.servlet.http.HttpServlet;
 8 | import javax.servlet.http.HttpServletRequest;
 9 | import javax.servlet.http.HttpServletResponse;
10 | import java.io.IOException;
11 | import java.io.PrintWriter;
12 | import java.util.ArrayList;
13 | 
14 | public class IndexListServlet extends HttpServlet {
15 | 	private ArrayList<Pair<String, String>> endpoints = new ArrayList<>();
16 | 	@Override
17 | 	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
18 | 		resp.setStatus(200);
19 | 		resp.setContentType("text/html");
20 | 
21 | 		PrintWriter writer = resp.getWriter();
22 | 		writer.println("<head><title>Maxwell's Daemon</title></head><body>");
23 | 		writer.println("<h1>Maxwell's Daemon</h1>");
24 | 		writer.println("<ul>");
25 | 
26 | 		for ( Pair<String, String> endpoint : endpoints ) {
27 | 			String li = String.format("\t<li><a href='%s'>%s</a> -- %s</li>", endpoint.getLeft(), endpoint.getLeft(), endpoint.getRight());
28 | 			writer.println(li);
29 | 		}
30 | 		writer.println("</ul>");
31 | 		writer.println("</body>");
32 | 	}
33 | 
34 | 	public void addLink(String endpoint, String description) {
35 | 		endpoints.add(Pair.of(endpoint, description));
36 | 
37 | 	}
38 | }
39 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/MaxwellDiagnostic.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import java.util.concurrent.CompletableFuture;
 4 | 
 5 | public interface MaxwellDiagnostic {
 6 | 
 7 | 	String getName();
 8 | 
 9 | 	boolean isMandatory();
10 | 
11 | 	String getResource();
12 | 
13 | 	CompletableFuture<MaxwellDiagnosticResult.Check> check();
14 | 
15 | }
16 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/MaxwellDiagnosticContext.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import java.util.List;
 4 | 
 5 | public class MaxwellDiagnosticContext {
 6 | 
 7 | 	public final Config config;
 8 | 	public final List<MaxwellDiagnostic> diagnostics;
 9 | 
10 | 	public MaxwellDiagnosticContext(Config config, List<MaxwellDiagnostic> diagnostics) {
11 | 		this.config = config;
12 | 		this.diagnostics = diagnostics;
13 | 	}
14 | 
15 | 	public static class Config {
16 | 		public boolean enable;
17 | 		public long timeout;
18 | 	}
19 | }
20 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/MaxwellDiagnosticResult.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import java.util.List;
 4 | import java.util.Map;
 5 | 
 6 | public class MaxwellDiagnosticResult {
 7 | 
 8 | 	private final boolean success;
 9 | 	private final boolean mandatoryFailed;
10 | 	private final List<Check> checks;
11 | 
12 | 	public MaxwellDiagnosticResult(List<Check> checks) {
13 | 		success = checks.stream().allMatch(Check::isSuccess);
14 | 		mandatoryFailed = checks.stream().anyMatch(check -> !check.success && check.mandatory);
15 | 		this.checks = checks;
16 | 	}
17 | 
18 | 	public boolean isSuccess() {
19 | 		return success;
20 | 	}
21 | 
22 | 	public boolean isMandatoryFailed() {
23 | 		return mandatoryFailed;
24 | 	}
25 | 
26 | 	public List<Check> getChecks() {
27 | 		return checks;
28 | 	}
29 | 
30 | 	public static class Check {
31 | 		private final String name;
32 | 		private final boolean success;
33 | 		private final boolean mandatory;
34 | 		private final String resource;
35 | 		private final Map<String, String> info;
36 | 
37 | 		public Check(MaxwellDiagnostic diagnostic, boolean success, Map<String, String> info) {
38 | 			this.name = diagnostic.getName();
39 | 			this.success = success;
40 | 			this.mandatory = diagnostic.isMandatory();
41 | 			this.resource = diagnostic.getResource();
42 | 			this.info = info;
43 | 		}
44 | 
45 | 		public String getName() {
46 | 			return name;
47 | 		}
48 | 
49 | 		public boolean isSuccess() {
50 | 			return success;
51 | 		}
52 | 
53 | 		public boolean isMandatory() {
54 | 			return mandatory;
55 | 		}
56 | 
57 | 		public String getResource() {
58 | 			return resource;
59 | 		}
60 | 
61 | 		public Map<String, String> getInfo() {
62 | 			return info;
63 | 		}
64 | 	}
65 | }
66 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHealthCheck.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import com.codahale.metrics.Meter;
 4 | import com.codahale.metrics.health.HealthCheck;
 5 | import com.zendesk.maxwell.producer.AbstractProducer;
 6 | 
 7 | 
 8 | public class MaxwellHealthCheck extends HealthCheck {
 9 | 	private final Meter failedMessageMeter;
10 | 
11 | 	public MaxwellHealthCheck(AbstractProducer producer) {
12 | 		this.failedMessageMeter = producer.getFailedMessageMeter();
13 | 	}
14 | 
15 | 	@Override
16 | 	protected Result check() throws Exception {
17 | 		// TODO: this should be configurable.
18 | 		if (failedMessageMeter != null && failedMessageMeter.getFifteenMinuteRate() > 0) {
19 | 			return Result.unhealthy(">1 messages failed to be sent to Kafka in the past 15minutes");
20 | 		} else {
21 | 			return Result.healthy();
22 | 		}
23 | 	}
24 | }
25 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHealthCheckFactory.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.monitoring;
2 | 
3 | import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
4 | import com.zendesk.maxwell.producer.AbstractProducer;
5 | 
6 | public interface MaxwellHealthCheckFactory {
7 | 	MaxwellHealthCheck createHealthCheck(AbstractProducer producer);
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/Metrics.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import com.codahale.metrics.Metric;
 4 | import com.codahale.metrics.MetricRegistry;
 5 | 
 6 | public interface Metrics {
 7 | 	String metricName(String... names);
 8 | 	MetricRegistry getRegistry();
 9 | 	<T extends Metric> void register(String name, T metric) throws IllegalArgumentException;
10 | }
11 | 
12 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/NoOpMetrics.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import com.codahale.metrics.Metric;
 4 | import com.codahale.metrics.MetricRegistry;
 5 | import com.codahale.metrics.health.HealthCheckRegistry;
 6 | 
 7 | public class NoOpMetrics implements Metrics {
 8 | 	public final MetricRegistry metricRegistry;
 9 | 	public final HealthCheckRegistry healthCheckRegistry;
10 | 
11 | 	public NoOpMetrics() {
12 | 		metricRegistry = new MetricRegistry();
13 | 		healthCheckRegistry = new HealthCheckRegistry();
14 | 	}
15 | 
16 | 	public String metricName(String... names) {
17 | 		return MetricRegistry.name("noop", names);
18 | 	}
19 | 
20 | 	public MetricRegistry getRegistry() {
21 | 		return metricRegistry;
22 | 	}
23 | 
24 | 	@Override
25 | 	public <T extends Metric> void register(String name, T metric) throws IllegalArgumentException {
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/monitoring/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * monitoring, diagnostics, and the HTTP server
3 |  */
4 | package com.zendesk.maxwell.monitoring;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * top level classes concerned with instantiating and running a Maxwell
3 |  * replicator.
4 |  */
5 | 
6 | package com.zendesk.maxwell;
7 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/AbstractProfilingProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | 
 6 | import java.io.File;
 7 | import java.io.FileOutputStream;
 8 | 
 9 | public abstract class AbstractProfilingProducer extends AbstractProducer {
10 | 	protected long count;
11 | 	protected long startTime;
12 | 	private FileOutputStream nullOutputStream;
13 | 
14 | 	public AbstractProfilingProducer(MaxwellContext context) {
15 | 		super(context);
16 | 		this.startTime = 0;
17 | 		this.count = 0;
18 | 	}
19 | 
20 | 	@Override
21 | 	public void push(RowMap r) throws Exception {
22 | 		if ( this.nullOutputStream == null ) {
23 | 			this.nullOutputStream = new FileOutputStream(new File("/dev/null"));
24 | 		}
25 | 
26 | 		if ( this.startTime == 0)
27 | 			this.startTime = System.currentTimeMillis();
28 | 
29 | 		String value = r.toJSON();
30 | 
31 | 		if (value != null) {
32 | 			nullOutputStream.write(value.getBytes());
33 | 		}
34 | 
35 | 		this.count++;
36 | 		this.context.setPosition(r);
37 | 	}
38 | }
39 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/BufferedProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.row.HeartbeatRowMap;
 5 | import com.zendesk.maxwell.row.RowMap;
 6 | 
 7 | import java.util.concurrent.LinkedBlockingQueue;
 8 | import java.util.concurrent.TimeUnit;
 9 | 
10 | public class BufferedProducer extends AbstractProducer {
11 | 	private final LinkedBlockingQueue<RowMap> queue;
12 | 
13 | 	public BufferedProducer(MaxwellContext context, int maxSize) {
14 | 		super(context);
15 | 		this.queue = new LinkedBlockingQueue<>(maxSize);
16 | 	}
17 | 
18 | 	@Override
19 | 	public void push(RowMap r) throws Exception {
20 | 		// set position on heartbeats immediately to ensure we terminate cleanly
21 | 		if (r instanceof HeartbeatRowMap) {
22 | 			this.context.setPosition(r);
23 | 		}
24 | 		try {
25 | 			this.queue.put(r);
26 | 		} catch ( InterruptedException e ) {}
27 | 	}
28 | 
29 | 	public RowMap poll(long timeout, TimeUnit unit) throws InterruptedException {
30 | 		RowMap r = this.queue.poll(timeout, unit);
31 | 		if (r != null) {
32 | 			this.context.setPosition(r);
33 | 		}
34 | 		return r;
35 | 	}
36 | }
37 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/EncryptionMode.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.producer;
2 | 
3 | public enum EncryptionMode {
4 | 	ENCRYPT_NONE,
5 | 	ENCRYPT_DATA,
6 | 	ENCRYPT_ALL
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/FileProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import java.io.File;
 4 | import java.io.FileWriter;
 5 | import java.io.IOException;
 6 | 
 7 | import com.zendesk.maxwell.MaxwellContext;
 8 | import com.zendesk.maxwell.row.RowMap;
 9 | 
10 | public class FileProducer extends AbstractProducer {
11 | 	private final File file;
12 | 	private final FileWriter fileWriter;
13 | 
14 | 	public FileProducer(MaxwellContext context, String filename) throws IOException {
15 | 		super(context);
16 | 		this.file = new File(filename);
17 | 		this.fileWriter = new FileWriter(this.file, true);
18 | 	}
19 | 
20 | 	@Override
21 | 	public void push(RowMap r) throws Exception {
22 | 		if ( !r.shouldOutput(outputConfig) ) {
23 | 			context.setPosition(r);
24 | 			return;
25 | 		}
26 | 
27 | 		String output = r.toJSON(outputConfig);
28 | 
29 | 		if ( output != null ) {
30 | 			this.fileWriter.write(r.toJSON(outputConfig));
31 | 			this.fileWriter.write('\n');
32 | 			this.fileWriter.flush();
33 | 		}
34 | 
35 | 		context.setPosition(r);
36 | 	}
37 | }
38 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/NoneProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | 
 6 | public class NoneProducer extends AbstractProducer {
 7 | 
 8 | 	public NoneProducer(MaxwellContext context) {
 9 | 		super(context);
10 | 	}
11 | 
12 | 	@Override
13 | 	public void push(RowMap r) throws Exception {
14 | 		this.context.setPosition(r);
15 | 	}
16 | }
17 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/ProducerFactory.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.producer;
2 | 
3 | import com.zendesk.maxwell.MaxwellContext;
4 | 
5 | public interface ProducerFactory {
6 | 	AbstractProducer createProducer(MaxwellContext context);
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/ProfilerProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | 
 6 | import java.io.File;
 7 | import java.io.FileOutputStream;
 8 | 
 9 | public class ProfilerProducer extends AbstractProfilingProducer {
10 | 	public ProfilerProducer(MaxwellContext context) {
11 | 		super(context);
12 | 	}
13 | 
14 | 	@Override
15 | 	public void push(RowMap r) throws Exception {
16 | 		super.push(r);
17 | 
18 | 		if ( this.count % 10000 == 0 ) {
19 | 			long elapsed = System.currentTimeMillis() - this.startTime;
20 | 			System.out.println("rows per second: " + (count * 1000) / elapsed);
21 | 		}
22 | 
23 | 		if ( this.count % 1000000 == 0 ) {
24 | 			System.out.println("resetting statistics.");
25 | 			this.count = 0;
26 | 			this.startTime = System.currentTimeMillis();
27 | 		}
28 | 	}
29 | }
30 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/StdoutProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | 
 6 | import java.util.concurrent.TimeUnit;
 7 | 
 8 | public class StdoutProducer extends AbstractProducer {
 9 | 	public StdoutProducer(MaxwellContext context) {
10 | 		super(context);
11 | 	}
12 | 
13 | 	@Override
14 | 	public void push(RowMap r) throws Exception {
15 | 		String output = r.toJSON(outputConfig);
16 | 
17 | 		if ( output != null && r.shouldOutput(outputConfig) )
18 | 			System.out.println(output);
19 | 
20 | 		this.messageLatencyTimer.update(
21 | 			System.currentTimeMillis() - r.getTimestampMillis(),
22 | 			TimeUnit.MILLISECONDS
23 | 		);
24 | 		this.context.setPosition(r);
25 | 	}
26 | }
27 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * producers write to streams and sinks of various types.
3 |  */
4 | package com.zendesk.maxwell.producer;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/HashFunction.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.producer.partitioners;
2 | 
3 | /**
4 |  * Created by kaufmannkr on 1/21/16.
5 |  */
6 | public interface HashFunction {
7 | 	int hashCode(String s);
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/HashFunctionDefault.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | /**
 4 |  * Created by kaufmannkr on 1/18/16.
 5 |  */
 6 | public class HashFunctionDefault implements HashFunction {
 7 | 	public int hashCode(String s) {
 8 | 		return s.hashCode();
 9 | 	}
10 | }
11 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/HashFunctionMurmur3.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.util.MurmurHash3;
 4 | 
 5 | /**
 6 |  * Created by kaufmannkr on 1/18/16.
 7 |  */
 8 | public class HashFunctionMurmur3 implements HashFunction {
 9 | 	private int seed;
10 | 	public HashFunctionMurmur3(int seed){
11 | 		this.seed = seed;
12 | 	}
13 | 	public int hashCode(String s) {
14 | 		return MurmurHash3.murmurhash3_x86_32(s, 0, s.length(), seed);
15 | 	}
16 | }
17 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellKafkaPartitioner.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | 
 5 | public class MaxwellKafkaPartitioner extends AbstractMaxwellPartitioner {
 6 | 	HashFunction hashFunc;
 7 | 
 8 | 	public MaxwellKafkaPartitioner(String hashFunction, String partitionKey, String csvPartitionColumns, String partitionKeyFallback) {
 9 | 		super(partitionKey, csvPartitionColumns, partitionKeyFallback);
10 | 
11 | 		int MURMUR_HASH_SEED = 25342;
12 | 		switch (hashFunction) {
13 | 			case "murmur3": this.hashFunc = new HashFunctionMurmur3(MURMUR_HASH_SEED);
14 | 				break;
15 | 			case "default":
16 | 			default:
17 | 				this.hashFunc = new HashFunctionDefault();
18 | 				break;
19 | 		}
20 | 	}
21 | 
22 | 	public int kafkaPartition(RowMap r, int numPartitions) {
23 | 		return Math.abs(hashFunc.hashCode(this.getHashString(r)) % numPartitions);
24 | 	}
25 | }
26 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellKinesisPartitioner.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | 
 5 | import org.apache.commons.codec.digest.DigestUtils;
 6 | 
 7 | public class MaxwellKinesisPartitioner extends AbstractMaxwellPartitioner {
 8 | 	boolean md5Keys;
 9 | 
10 | 	public MaxwellKinesisPartitioner(String partitionKey, String csvPartitionColumns, String partitionKeyFallback, boolean md5Keys) {
11 | 		super(partitionKey, csvPartitionColumns, partitionKeyFallback);
12 | 
13 | 		this.md5Keys = md5Keys;
14 | 	}
15 | 
16 | 	public String getKinesisKey(RowMap r) {
17 | 		String key = this.getHashString(r);
18 | 
19 | 		if(md5Keys) {
20 | 			return DigestUtils.md5Hex(key);
21 | 		}
22 | 
23 | 		return key;
24 | 	}
25 | }
26 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSNSPartitioner.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | 
 5 | public class MaxwellSNSPartitioner extends AbstractMaxwellPartitioner {
 6 | 
 7 |     public MaxwellSNSPartitioner(String partitionKey, String csvPartitionColumns, String partitionKeyFallback) {
 8 |         super(partitionKey, csvPartitionColumns, partitionKeyFallback);
 9 |     }
10 | 
11 |     public String getSNSKey(RowMap r) {
12 |         String key = this.getHashString(r);
13 |         return key;
14 |     }
15 | }


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSQSPartitioner.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | 
 5 | public class MaxwellSQSPartitioner extends AbstractMaxwellPartitioner {
 6 | 
 7 |     public MaxwellSQSPartitioner(String partitionKey, String csvPartitionColumns, String partitionKeyFallback) {
 8 |         super(partitionKey, csvPartitionColumns, partitionKeyFallback);
 9 |     }
10 | 
11 |     public String getSQSKey(RowMap r) {
12 |         String key = this.getHashString(r);
13 |         return key;
14 |     }
15 | }


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/PartitionBy.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | public enum PartitionBy {
 4 | 	DATABASE,
 5 | 	TABLE,
 6 | 	PRIMARY_KEY,
 7 | 	TRANSACTION_ID,
 8 | 	THREAD_ID,
 9 | 	COLUMN,
10 | 	RANDOM
11 | }
12 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/producer/partitioners/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * partitioners deal with subdividing streams by data key
3 |  */
4 | package com.zendesk.maxwell.producer.partitioners;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/recovery/RecoveryFilter.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.recovery;
 2 | 
 3 | import com.zendesk.maxwell.filtering.Filter;
 4 | 
 5 | /**
 6 |  * filter out (via a blacklist) everything except for `maxwell`.`positions`.
 7 |  * this makes a possibly out of sync schema harmless.
 8 |  */
 9 | public class RecoveryFilter extends Filter {
10 | 	private final String maxwellDatabaseName;
11 | 
12 | 	public RecoveryFilter(String maxwellDatabaseName) {
13 | 		this.maxwellDatabaseName = maxwellDatabaseName;
14 | 	}
15 | 
16 | 	@Override
17 | 	public boolean isTableBlacklisted(String databaseName, String tableName) {
18 | 		return !(databaseName.equals(maxwellDatabaseName) && tableName.equals("heartbeats"));
19 | 	}
20 | }
21 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/recovery/RecoveryInfo.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.recovery;
 2 | 
 3 | import com.zendesk.maxwell.replication.Position;
 4 | 
 5 | public class RecoveryInfo {
 6 | 	public Position position;
 7 | 	public Long serverID;
 8 | 	public String clientID;
 9 | 
10 | 	public RecoveryInfo(Position position, Long serverID, String clientID) {
11 | 		this.position = position;
12 | 		this.serverID = serverID;
13 | 		this.clientID = clientID;
14 | 	}
15 | 
16 | 	public long getHeartbeat() {
17 | 		return position.getLastHeartbeatRead();
18 | 	}
19 | 
20 | 	public String toString() {
21 | 		return "<RecoveryInfo" +
22 | 				" position: " + position +
23 | 				", serverId: " + serverID +
24 | 				", clientId: " + clientID +
25 | 				">";
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/recovery/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * allows users to continue a stream after changing masters
3 |  */
4 | package com.zendesk.maxwell.recovery;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/replication/HeartbeatNotifier.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.replication;
 2 | 
 3 | import java.util.Observable;
 4 | 
 5 | public class HeartbeatNotifier extends Observable {
 6 | 
 7 | 	protected void heartbeat(long heartbeat) {
 8 | 		setChanged();
 9 | 		notifyObservers(heartbeat);
10 | 	}
11 | 
12 | }
13 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/replication/Replicator.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.replication;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | import com.zendesk.maxwell.schema.SchemaStoreException;
 5 | import com.zendesk.maxwell.schema.Schema;
 6 | import com.zendesk.maxwell.util.StoppableTask;
 7 | 
 8 | /**
 9 |  * Created by ben on 10/23/16.
10 |  */
11 | public interface Replicator extends StoppableTask {
12 | 	void startReplicator() throws Exception;
13 | 	RowMap getRow() throws Exception;
14 | 	Long getLastHeartbeatRead();
15 | 	Schema getSchema() throws SchemaStoreException;
16 | 	Long getSchemaId() throws SchemaStoreException;
17 | 
18 | 	void stopAtHeartbeat(long heartbeat);
19 | 	void runLoop() throws Exception;
20 | }
21 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/replication/ReplicatorNotReadyException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.replication;
2 | 
3 | public class ReplicatorNotReadyException extends Exception {
4 | 	public ReplicatorNotReadyException(String message) {
5 | 		super(message);
6 | 	}
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/replication/TableCache.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.replication;
 2 | 
 3 | import java.util.HashMap;
 4 | 
 5 | import com.zendesk.maxwell.filtering.Filter;
 6 | import com.zendesk.maxwell.schema.Database;
 7 | import com.zendesk.maxwell.schema.Schema;
 8 | import com.zendesk.maxwell.schema.Table;
 9 | 
10 | public class TableCache {
11 | 	private final String maxwellDB;
12 | 
13 | 	public TableCache(String maxwellDB) {
14 | 		this.maxwellDB = maxwellDB;
15 | 	}
16 | 	private final HashMap<Long, Table> tableMapCache = new HashMap<>();
17 | 
18 | 	public void processEvent(Schema schema, Filter filter, Boolean ignoreMissingSchema, Long tableId, String dbName, String tblName) {
19 | 		if ( !tableMapCache.containsKey(tableId)) {
20 | 			if ( filter.isTableBlacklisted(dbName, tblName) ) {
21 | 				return;
22 | 			}
23 | 
24 | 
25 | 			Database db = schema.findDatabase(dbName);
26 | 			if ( db == null ) {
27 | 				if ( !ignoreMissingSchema || filter.includes(dbName, tblName) )
28 | 					throw new RuntimeException("Couldn't find database " + dbName);
29 | 
30 | 			} else {
31 | 				Table tbl = db.findTable(tblName);
32 | 
33 | 				if (tbl == null) {
34 | 					if ( !ignoreMissingSchema || filter.includes(dbName, tblName) )
35 | 						throw new RuntimeException("Couldn't find table " + tblName + " in database " + dbName);
36 | 
37 | 				} else {
38 | 					tableMapCache.put(tableId, tbl);
39 | 				}
40 | 			}
41 | 		}
42 | 
43 | 	}
44 | 
45 | 	public Table getTable(Long tableId) {
46 | 		return tableMapCache.get(tableId);
47 | 	}
48 | 
49 | 	public void clear() {
50 | 		tableMapCache.clear();
51 | 	}
52 | }
53 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/replication/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * concerns the operation of the binlog replicator
3 |  */
4 | package com.zendesk.maxwell.replication;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/row/EncryptionContext.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.row;
 2 | 
 3 | import java.security.NoSuchAlgorithmException;
 4 | import java.security.SecureRandom;
 5 | 
 6 | class EncryptionContext {
 7 | 	String secretKey;
 8 | 	byte[] iv;
 9 | 
10 | 	EncryptionContext(String secretKey, byte[] iv) {
11 | 		this.secretKey = secretKey;
12 | 		this.iv = iv;
13 | 	}
14 | 
15 | 	public static EncryptionContext create(String secretKey) throws NoSuchAlgorithmException {
16 | 		SecureRandom randomSecureRandom = SecureRandom.getInstance("SHA1PRNG");
17 | 		byte[] iv = new byte[16];
18 | 		randomSecureRandom.nextBytes(iv);
19 | 		return new EncryptionContext(secretKey, iv);
20 | 	}
21 | }
22 | 
23 | 
24 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/row/HeartbeatRowMap.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.row;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | import com.zendesk.maxwell.replication.BinlogPosition;
 5 | import com.zendesk.maxwell.replication.Position;
 6 | 
 7 | import java.io.IOException;
 8 | import java.util.ArrayList;
 9 | 
10 | /**
11 |  * Created by ben on 9/7/16.
12 |  */
13 | public class HeartbeatRowMap extends RowMap {
14 | 	public HeartbeatRowMap(String database, Position position, Position nextPosition) {
15 | 		super("heartbeat", database, "heartbeats", position.getLastHeartbeatRead(), new ArrayList<String>(), position, nextPosition, null);
16 | 	}
17 | 
18 | 	public static HeartbeatRowMap valueOf(String database, Position position, Position nextPosition) {
19 | 		return new HeartbeatRowMap(database, position, nextPosition);
20 | 	}
21 | 
22 | 	@Override
23 | 	public String toJSON(MaxwellOutputConfig outputConfig) throws IOException {
24 | 		return null;
25 | 	}
26 | 
27 | 	@Override
28 | 	public String toJSON() throws IOException {
29 | 		return null;
30 | 	}
31 | 
32 | 	@Override
33 | 	public boolean isTXCommit() {
34 | 		return true;
35 | 	}
36 | 
37 | 	@Override
38 | 	public boolean shouldOutput(MaxwellOutputConfig outputConfig) {
39 | 		return false;
40 | 	}
41 | }
42 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/row/RawJSONString.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.row;
 2 | 
 3 | // we wrap up raw json here and use the type of this class to allow
 4 | // tunneling pre-serialized JSON data through a RowMap
 5 | 
 6 | import java.io.Serializable;
 7 | import java.util.Objects;
 8 | 
 9 | public class RawJSONString implements Serializable {
10 | 	private static final long serialVersionUID = -5600187114417848732L;
11 | 
12 | 	public final String json;
13 | 
14 | 	public RawJSONString(String json) {
15 | 		this.json = json;
16 | 	}
17 | 
18 | 	@Override
19 | 	public boolean equals(Object that) {
20 | 		if ( !(that instanceof RawJSONString) )
21 | 			return false;
22 | 
23 | 		return Objects.equals(this.json, ((RawJSONString) that).json);
24 | 	}
25 | }
26 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/row/RowEncrypt.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.row;
 2 | 
 3 | import javax.crypto.Cipher;
 4 | import javax.crypto.spec.IvParameterSpec;
 5 | import javax.crypto.spec.SecretKeySpec;
 6 | 
 7 | import org.apache.commons.codec.binary.Base64;
 8 | 
 9 | import java.io.IOException;
10 | import java.nio.charset.Charset;
11 | 
12 | public class RowEncrypt {
13 | 	private final static String TEXT_ENCODING = "UTF-8";
14 | 	private final static String BYTE_ENCODING = "ASCII";
15 | 
16 | 	public static String encrypt(String value, String secretKey, byte[] initVector) throws Exception {
17 | 		IvParameterSpec ivSpec = new IvParameterSpec(initVector);
18 | 		Cipher cipher = getCipher();
19 | 		cipher.init(Cipher.ENCRYPT_MODE, loadKey(secretKey), ivSpec);
20 | 
21 | 		byte[] encrypted = cipher.doFinal(value.getBytes(TEXT_ENCODING));
22 | 		return Base64.encodeBase64String(encrypted);
23 | 	}
24 | 
25 | 	public static String decrypt(String value, String secretKey, String initVector) throws Exception {
26 | 		IvParameterSpec ivSpec = new IvParameterSpec(base64Decode(initVector));
27 | 		Cipher cipher = getCipher();
28 | 		cipher.init(Cipher.DECRYPT_MODE, loadKey(secretKey), ivSpec);
29 | 
30 | 		return new String(cipher.doFinal(base64Decode(value)), Charset.forName(TEXT_ENCODING));
31 | 	}
32 | 
33 | 	private static Cipher getCipher() throws Exception {
34 | 		return Cipher.getInstance("AES/CBC/PKCS5PADDING");
35 | 	}
36 | 
37 | 	private static SecretKeySpec loadKey(String secretKey) throws IOException {
38 | 		return new SecretKeySpec(secretKey.getBytes(TEXT_ENCODING), "AES");
39 | 	}
40 | 
41 | 	private static byte[] base64Decode(String value) throws IOException {
42 | 		return Base64.decodeBase64(value.getBytes(BYTE_ENCODING));
43 | 	}
44 | }
45 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/row/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * Maxwell's representation of a row from the binlog
3 |  */
4 | package com.zendesk.maxwell.row;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ReadOnlyMysqlPositionStore.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema;
 2 | 
 3 | import com.zendesk.maxwell.replication.Position;
 4 | import com.zendesk.maxwell.util.ConnectionPool;
 5 | 
 6 | /**
 7 |  * a schema position object that doesn't write its position out.
 8 |  * useful for "replay" mode.
 9 |  */
10 | public class ReadOnlyMysqlPositionStore extends MysqlPositionStore {
11 | 	public ReadOnlyMysqlPositionStore(ConnectionPool pool, Long serverID, String clientID, boolean gtidMode) {
12 | 		super(pool, serverID, clientID, gtidMode);
13 | 	}
14 | 
15 | 	@Override
16 | 	public void set(Position p) { }
17 | 
18 | 	@Override
19 | 	public long heartbeat() throws Exception {
20 | 		return System.currentTimeMillis();
21 | 	}
22 | }
23 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/SchemaStoreException.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.schema;
2 | 
3 | public class SchemaStoreException extends Exception {
4 | 	public SchemaStoreException (String message) { super(message); }
5 | 	public SchemaStoreException (Exception e) { super(e); }
6 | 	private static final long serialVersionUID = 1L;
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/ColumnDefCastException.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | public class ColumnDefCastException extends Exception {
 4 | 	public final Object givenValue;
 5 | 	public final ColumnDef def;
 6 | 
 7 | 	public String database;
 8 | 	public String table;
 9 | 
10 | 	public ColumnDefCastException(ColumnDef def, Object givenValue) {
11 | 		super();
12 | 		this.def = def;
13 | 		this.givenValue = givenValue;
14 | 	}
15 | }
16 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/ColumnDefDeserializer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.fasterxml.jackson.core.JsonParser;
 4 | import com.fasterxml.jackson.core.JsonProcessingException;
 5 | import com.fasterxml.jackson.databind.DeserializationContext;
 6 | import com.fasterxml.jackson.databind.JsonDeserializer;
 7 | import com.fasterxml.jackson.databind.JsonNode;
 8 | 
 9 | import java.util.*;
10 | 
11 | import java.io.IOException;
12 | 
13 | public class ColumnDefDeserializer extends JsonDeserializer<ColumnDef> {
14 | 	@Override
15 | 	public ColumnDef deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
16 | 		JsonNode node = jp.getCodec().readTree(jp);
17 | 
18 | 		String type    = node.get("type").asText();
19 | 		String name    = node.get("name").asText();
20 | 		String charset = node.path("charset").asText();
21 | 
22 | 		boolean signed = node.path("signed").asBoolean(false);
23 | 
24 | 		String[] enumValues = null;
25 | 		JsonNode enumValueNode = node.get("enum-values");
26 | 		if ( enumValueNode != null ) {
27 | 			ArrayList<String> values = new ArrayList<>();
28 | 			for ( JsonNode n : enumValueNode )
29 | 				values.add(n.asText());
30 | 
31 | 			enumValues = (String[]) values.toArray(new String[0]);
32 | 		}
33 | 
34 | 		Long columnLength = null;
35 | 		JsonNode columnLengthNode = node.get("column-length");
36 | 		if ( columnLengthNode != null ) {
37 | 			columnLength = columnLengthNode.asLong();
38 | 		}
39 | 		return ColumnDef.build(name, charset, type, (short) 0, signed, enumValues, columnLength);
40 | 	}
41 | }
42 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/DateColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | 
 5 | public class DateColumnDef extends ColumnDef {
 6 | 	private DateColumnDef(String name, String type, short pos) {
 7 | 		super(name, type, pos);
 8 | 	}
 9 | 
10 | 	public static DateColumnDef create(String name, String type, short pos) {
11 | 		DateColumnDef temp = new DateColumnDef(name, type, pos);
12 | 		return (DateColumnDef) INTERNER.intern(temp);
13 | 	}
14 | 
15 | 	@Override
16 | 	public String toSQL(Object value) {
17 | 		String formatted = DateFormatter.formatDate(value);
18 | 		if ( formatted == null )
19 | 			return null;
20 | 		else
21 | 			return "'" +  formatted + "'";
22 | 	}
23 | 
24 | 	@Override
25 | 	public Object asJSON(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
26 | 		if ( value instanceof String ) {
27 | 			// bootstrapper just gives up on bothering with date processing
28 | 			if ( config.zeroDatesAsNull && "0000-00-00".equals((String) value) )
29 | 				return null;
30 | 			else
31 | 				return value;
32 | 		} else if ( value instanceof Long && (Long) value == Long.MIN_VALUE ) {
33 | 			if ( config.zeroDatesAsNull )
34 | 				return null;
35 | 			else
36 | 				return "0000-00-00";
37 | 		}
38 | 
39 | 		try {
40 | 			return DateFormatter.formatDate(value);
41 | 		} catch ( IllegalArgumentException e ) {
42 | 			throw new ColumnDefCastException(this, value);
43 | 		}
44 | 	}
45 | }
46 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/DateTimeColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | 
 5 | import java.sql.Timestamp;
 6 | 
 7 | public class DateTimeColumnDef extends ColumnDefWithLength {
 8 | 
 9 | 	private final boolean isTimestamp = getType().equals("timestamp");
10 | 
11 | 	private DateTimeColumnDef(String name, String type, short pos, Long columnLength) {
12 | 		super(name, type, pos, columnLength);
13 | 	}
14 | 
15 | 	public static DateTimeColumnDef create(String name, String type, short pos, Long columnLength) {
16 | 		DateTimeColumnDef temp = new DateTimeColumnDef(name, type, pos, columnLength);
17 | 		return (DateTimeColumnDef) INTERNER.intern(temp);
18 | 	}
19 | 
20 | 	protected String formatValue(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
21 | 		// special case for those broken mysql dates.
22 | 		if ( value instanceof Long ) {
23 | 			Long v = (Long) value;
24 | 			if ( v == Long.MIN_VALUE || (v == 0L && isTimestamp) ) {
25 | 				if ( config.zeroDatesAsNull )
26 | 					return null;
27 | 				else
28 | 					return appendFractionalSeconds("0000-00-00 00:00:00", 0, getColumnLength());
29 | 			}
30 | 		}
31 | 
32 | 		try {
33 | 			Timestamp ts = DateFormatter.extractTimestamp(value);
34 | 			String dateString = DateFormatter.formatDateTime(value, ts);
35 | 			return appendFractionalSeconds(dateString, ts.getNanos(), getColumnLength());
36 | 		} catch ( IllegalArgumentException e ) {
37 | 			throw new ColumnDefCastException(this, value);
38 | 		}
39 | 	}
40 | }
41 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/DecimalColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import java.math.BigDecimal;
 4 | 
 5 | public class DecimalColumnDef extends ColumnDef {
 6 | 	private DecimalColumnDef(String name, String type, short pos) {
 7 | 		super(name, type, pos);
 8 | 	}
 9 | 
10 | 	public static DecimalColumnDef create(String name, String type, short pos) {
11 | 		DecimalColumnDef temp = new DecimalColumnDef(name, type, pos);
12 | 		return (DecimalColumnDef) INTERNER.intern(temp);
13 | 	}
14 | 
15 | 	@Override
16 | 	public String toSQL(Object value) {
17 | 		BigDecimal d = (BigDecimal) value;
18 | 
19 | 		return d.toEngineeringString();
20 | 	}
21 | }
22 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/EnumColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | 
 5 | public class EnumColumnDef extends EnumeratedColumnDef {
 6 | 	private EnumColumnDef(String name, String type, short pos, String[] enumValues) {
 7 | 		super(name, type, pos, enumValues);
 8 | 	}
 9 | 
10 | 	public static EnumColumnDef create(String name, String type, short pos, String[] enumValues) {
11 | 		EnumColumnDef temp = new EnumColumnDef(name, type, pos, enumValues);
12 | 		return (EnumColumnDef) INTERNER.intern(temp);
13 | 	}
14 | 
15 | 	@Override
16 | 	public String toSQL(Object value) throws ColumnDefCastException {
17 | 		return "'" + asString(value) + "'";
18 | 	}
19 | 
20 | 	@Override
21 | 	public String asJSON(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
22 | 		return asString(value);
23 | 	}
24 | 
25 | 	private String asString(Object value) throws ColumnDefCastException {
26 | 		if ( value instanceof String ) {
27 | 			return ( String ) value;
28 | 		} else if ( value instanceof Integer ) {
29 | 			Integer i = (Integer) value;
30 | 
31 | 			if (i == 0)
32 | 				return null;
33 | 			else
34 | 				return getEnumValues().get(((Integer) value) - 1);
35 | 		} else {
36 | 			throw new ColumnDefCastException(this, value);
37 | 		}
38 | 	}
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/EnumeratedColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.fasterxml.jackson.annotation.JsonProperty;
 4 | import com.google.common.collect.ImmutableList;
 5 | 
 6 | import java.util.List;
 7 | import java.util.Objects;
 8 | 
 9 | abstract public class EnumeratedColumnDef extends ColumnDef  {
10 | 	@JsonProperty("enum-values")
11 | 	private final List<String> enumValues;
12 | 
13 | 	protected EnumeratedColumnDef(String name, String type, short pos, String [] enumValues) {
14 | 		super(name, type, pos);
15 | 		ImmutableList.Builder<String> builder = ImmutableList.builderWithExpectedSize(enumValues.length);
16 | 		for (String enumValue : enumValues) {
17 | 			builder.add(enumValue.intern());
18 | 		}
19 | 		this.enumValues = builder.build();
20 | 	}
21 | 
22 | 	public List<String> getEnumValues() {
23 | 		return enumValues;
24 | 	}
25 | 
26 | 	@Override
27 | 	public boolean equals(Object o) {
28 | 		if (o.getClass() == getClass()) {
29 | 			EnumeratedColumnDef other = (EnumeratedColumnDef)o;
30 | 			return super.equals(o)
31 | 					&& Objects.equals(enumValues, other.enumValues);
32 | 		}
33 | 		return false;
34 | 	}
35 | 
36 | 	@Override
37 | 	public int hashCode() {
38 | 		int hash = super.hashCode();
39 | 		return 31 * hash + Objects.hash(enumValues);
40 | 	}
41 | }
42 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/FloatColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | public class FloatColumnDef extends ColumnDef {
 4 | 	private FloatColumnDef(String name, String type, short pos) {
 5 | 		super(name, type, pos);
 6 | 	}
 7 | 
 8 | 	public static FloatColumnDef create(String name, String type, short pos) {
 9 | 		FloatColumnDef temp = new FloatColumnDef(name, type, pos);
10 | 		return (FloatColumnDef) INTERNER.intern(temp);
11 | 	}
12 | 
13 | 	@Override
14 | 	public String toSQL(Object value) {
15 | 		return value.toString();
16 | 	}
17 | }
18 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/GeometryColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.vividsolutions.jts.geom.Geometry;
 4 | import com.vividsolutions.jts.io.ParseException;
 5 | import com.vividsolutions.jts.io.WKBReader;
 6 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 7 | 
 8 | import java.util.Arrays;
 9 | 
10 | /**
11 |  * Created by ben on 12/30/15.
12 |  */
13 | public class GeometryColumnDef extends ColumnDef {
14 | 	private GeometryColumnDef(String name, String type, short pos) {
15 | 		super(name, type, pos);
16 | 	}
17 | 
18 | 	public static GeometryColumnDef create(String name, String type, short pos) {
19 | 		GeometryColumnDef temp = new GeometryColumnDef(name, type, pos);
20 | 		return (GeometryColumnDef) INTERNER.intern(temp);
21 | 	}
22 | 
23 | 	@Override
24 | 	public Object asJSON(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
25 | 		Geometry geometry = null;
26 | 		if ( value instanceof Geometry ) {
27 | 			geometry = (Geometry) value;
28 | 		} else if ( value instanceof byte[] ) {
29 | 			byte []bytes = (byte[]) value;
30 | 
31 | 			// mysql sprinkles 4 mystery bytes on top of the GIS data.
32 | 			bytes = Arrays.copyOfRange(bytes, 4, bytes.length);
33 | 			final WKBReader reader = new WKBReader();
34 | 
35 | 			try {
36 | 				geometry = reader.read(bytes);
37 | 			} catch ( ParseException e ) {
38 | 				throw new RuntimeException("Could not parse geometry: " + e);
39 | 			}
40 | 
41 | 		} else {
42 | 			throw new ColumnDefCastException(this, value);
43 | 		}
44 | 
45 | 		return geometry.toText();
46 | 	}
47 | 
48 | 	@Override
49 | 	public String toSQL(Object value) {
50 | 		return null;
51 | 	}
52 | }
53 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/JsonColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary;
 4 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 5 | import com.zendesk.maxwell.row.RawJSONString;
 6 | 
 7 | import java.io.IOException;
 8 | 
 9 | public class JsonColumnDef extends ColumnDef {
10 | 	private JsonColumnDef(String name, String type, short pos) {
11 | 		super(name, type, pos);
12 | 	}
13 | 
14 | 	public static JsonColumnDef create(String name, String type, short pos) {
15 | 		JsonColumnDef temp = new JsonColumnDef(name, type, pos);
16 | 		return (JsonColumnDef) INTERNER.intern(temp);
17 | 	}
18 | 
19 | 	@Override
20 | 	public Object asJSON(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
21 | 		String jsonString;
22 | 
23 | 		if ( value instanceof String ) {
24 | 			return new RawJSONString((String) value);
25 | 		} else if ( value instanceof byte[] ){
26 | 			try {
27 | 				byte[] bytes = (byte[]) value;
28 | 				jsonString = bytes.length > 0 ? JsonBinary.parseAsString(bytes) : "null";
29 | 				return new RawJSONString(jsonString);
30 | 			} catch (IOException e) {
31 | 				throw new RuntimeException(e);
32 | 			}
33 | 		} else {
34 | 			throw new ColumnDefCastException(this, value);
35 | 		}
36 | 	}
37 | 
38 | 	@Override
39 | 	public String toSQL(Object value) {
40 | 		return null;
41 | 	}
42 | }
43 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/SetColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import java.util.ArrayList;
 4 | import java.util.Arrays;
 5 | import java.util.List;
 6 | 
 7 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 8 | import org.apache.commons.lang3.StringUtils;
 9 | 
10 | public class SetColumnDef extends EnumeratedColumnDef {
11 | 	private SetColumnDef(String name, String type, short pos, String[] enumValues) {
12 | 		super(name, type, pos, enumValues);
13 | 	}
14 | 
15 | 	public static SetColumnDef create(String name, String type, short pos, String[] enumValues) {
16 | 		SetColumnDef temp = new SetColumnDef(name, type, pos, enumValues);
17 | 		return (SetColumnDef) INTERNER.intern(temp);
18 | 	}
19 | 
20 | 	@Override
21 | 	public String toSQL(Object value) throws ColumnDefCastException {
22 | 		return "'" + StringUtils.join(asList(value), "'") + "'";
23 | 	}
24 | 
25 | 	@Override
26 | 	public Object asJSON(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
27 | 		return asList(value);
28 | 	}
29 | 
30 | 	private ArrayList<String> asList(Object value) throws ColumnDefCastException {
31 | 		if ( value instanceof String ) {
32 | 			return new ArrayList<>(Arrays.asList((( String ) value).split(",")));
33 | 		} else if ( value instanceof Long ) {
34 | 			ArrayList<String> values = new ArrayList<>();
35 | 			long v = (Long) value;
36 | 			List<String> enumValues = getEnumValues();
37 | 			for (int i = 0; i < enumValues.size(); i++) {
38 | 				if (((v >> i) & 1) == 1) {
39 | 					values.add(enumValues.get(i));
40 | 				}
41 | 			}
42 | 			return values;
43 | 		} else {
44 | 			throw new ColumnDefCastException(this, value);
45 | 		}
46 | 	}
47 | }
48 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/TimeColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | 
 5 | import java.sql.Time;
 6 | import java.sql.Timestamp;
 7 | 
 8 | public class TimeColumnDef extends ColumnDefWithLength {
 9 | 	private TimeColumnDef(String name, String type, short pos, Long columnLength) {
10 | 		super(name, type, pos, columnLength);
11 | 	}
12 | 
13 | 	public static TimeColumnDef create(String name, String type, short pos, Long columnLength) {
14 | 		TimeColumnDef temp = new TimeColumnDef(name, type, pos, columnLength);
15 | 		return (TimeColumnDef) INTERNER.intern(temp);
16 | 	}
17 | 
18 | 	protected String formatValue(Object value, MaxwellOutputConfig config) throws ColumnDefCastException {
19 | 		if ( value instanceof Timestamp ) {
20 | 			Time time = new Time(((Timestamp) value).getTime());
21 | 			String timeAsStr = String.valueOf(time);
22 | 
23 | 			return appendFractionalSeconds(timeAsStr, ((Timestamp) value).getNanos(), this.getColumnLength());
24 | 
25 | 		} else if ( value instanceof Long ) {
26 | 			Time time = new Time((Long) value / 1000);
27 | 			String timeAsStr = String.valueOf(time);
28 | 
29 | 			return appendFractionalSeconds(timeAsStr, (int) ((Long) value % 1000000) * 1000, this.getColumnLength());
30 | 		} else if ( value instanceof Time ){
31 | 			return String.valueOf((Time) value);
32 | 		} else {
33 | 			throw new ColumnDefCastException(this, value);
34 | 		}
35 | 	}
36 | }
37 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/YearColumnDef.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.columndef;
 2 | 
 3 | import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 4 | 
 5 | import java.sql.Date;
 6 | import java.util.Calendar;
 7 | 
 8 | public class YearColumnDef extends ColumnDef {
 9 | 	private YearColumnDef(String name, String type, short pos) {
10 | 		super(name, type, pos);
11 | 	}
12 | 
13 | 	public static YearColumnDef create(String name, String type, short pos) {
14 | 		YearColumnDef temp = new YearColumnDef(name, type, pos);
15 | 		return (YearColumnDef) INTERNER.intern(temp);
16 | 	}
17 | 
18 | 	@Override
19 | 	public Object asJSON(Object value, MaxwellOutputConfig outputConfig) {
20 | 		if ( value instanceof Date ) {
21 | 			Calendar calendar = Calendar.getInstance();
22 | 			calendar.setTime(( java.sql.Date ) value);
23 | 			return calendar.get(Calendar.YEAR);
24 | 		}
25 | 		return value;
26 | 	}
27 | 
28 | 	@Override
29 | 	public String toSQL(Object value) {
30 | 		return ((Integer)value).toString();
31 | 	}
32 | 
33 | }
34 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/columndef/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * column type definitions - cast from a mysql column to JSON
3 |  */
4 | package com.zendesk.maxwell.schema.columndef;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/.ruby-version:
--------------------------------------------------------------------------------
1 | 2.1.5
2 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/AddColumnMod.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | import com.zendesk.maxwell.schema.columndef.ColumnDef;
 5 | 
 6 | import java.util.List;
 7 | 
 8 | class AddColumnMod extends ColumnMod {
 9 | 	public ColumnDef definition;
10 | 	public ColumnPosition position;
11 | 
12 | 	public AddColumnMod(String name, ColumnDef d, ColumnPosition position) {
13 | 		super(name);
14 | 		this.definition = d;
15 | 		this.position = position;
16 | 	}
17 | 
18 | 	@Override
19 | 	public void apply(Table table, List<DeferredPositionUpdate> deferred) throws InvalidSchemaError {
20 | 		int index = position.index(table, null);
21 | 
22 | 		if ( index == ColumnPosition.AFTER_NOT_FOUND) {
23 | 			deferred.add(new DeferredPositionUpdate(definition.getName(), position));
24 | 			index = 0;
25 | 		}
26 | 
27 | 		table.addColumn(index, this.definition);
28 | 
29 | 	}
30 | }
31 | 
32 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ChangeColumnMod.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | import com.zendesk.maxwell.schema.columndef.ColumnDef;
 5 | 
 6 | import java.util.List;
 7 | 
 8 | class ChangeColumnMod extends ColumnMod {
 9 | 	public ColumnDef definition;
10 | 	public ColumnPosition position;
11 | 
12 | 	public ChangeColumnMod(String name, ColumnDef d, ColumnPosition position ) {
13 | 		super(name);
14 | 		this.definition = d;
15 | 		this.position = position;
16 | 	}
17 | 
18 | 	@Override
19 | 	public void apply(Table table, List<DeferredPositionUpdate> deferred) throws InvalidSchemaError {
20 | 		int idx = originalIndex(table);
21 | 		table.changeColumn(idx, position, definition, deferred);
22 | 	}
23 | }
24 | 
25 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ColumnMod.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | abstract class ColumnMod {
 8 | 	public String name;
 9 | 
10 | 	public ColumnMod(String name) {
11 | 		this.name = name;
12 | 	}
13 | 
14 | 	protected int originalIndex(Table table) throws InvalidSchemaError {
15 | 		int originalIndex = table.findColumnIndex(name);
16 | 
17 | 		if ( originalIndex == -1 )
18 | 			throw new InvalidSchemaError("Could not find column " + name + " in " + table.getName());
19 | 
20 | 		return originalIndex;
21 | 	}
22 | 
23 | 	public abstract void apply(Table table, List<DeferredPositionUpdate> deferred) throws InvalidSchemaError;
24 | }
25 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ColumnPosition.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | 
 5 | public class ColumnPosition {
 6 | 	enum Position { FIRST, AFTER, DEFAULT };
 7 | 	public static final int AFTER_NOT_FOUND = -999;
 8 | 
 9 | 	public Position position;
10 | 	public String afterColumn;
11 | 
12 | 	public ColumnPosition() {
13 | 		this.position = Position.DEFAULT;
14 | 	}
15 | 
16 | 	public int index(Table t, Integer defaultIndex) {
17 | 		switch(position) {
18 | 		case FIRST:
19 | 			return 0;
20 | 		case DEFAULT:
21 | 			if ( defaultIndex != null )
22 | 				return defaultIndex;
23 | 			else
24 | 				return t.getColumnList().size();
25 | 
26 | 		case AFTER:
27 | 			int afterIdx = t.findColumnIndex(afterColumn);
28 | 
29 | 			// see issue #1216
30 | 			if ( afterIdx == -1 )
31 | 				return AFTER_NOT_FOUND;
32 | 
33 | 			return afterIdx + 1;
34 | 		}
35 | 		return -1;
36 | 	}
37 | }
38 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/DatabaseAlter.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.filtering.Filter;
 4 | import com.zendesk.maxwell.schema.*;
 5 | 
 6 | public class DatabaseAlter extends SchemaChange {
 7 | 	public String database;
 8 | 	public String charset;
 9 | 
10 | 	public DatabaseAlter(String database) {
11 | 		this.database = database;
12 | 	}
13 | 
14 | 	@Override
15 | 	public ResolvedDatabaseAlter resolve(Schema s) throws InvalidSchemaError {
16 | 		return new ResolvedDatabaseAlter(this.database, this.charset);
17 | 	}
18 | 
19 | 	@Override
20 | 	public boolean isBlacklisted(Filter filter) {
21 | 		if ( filter == null ) {
22 | 			return false;
23 | 		} else {
24 | 			return filter.isDatabaseBlacklisted(database);
25 | 		}
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/DatabaseCreate.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.filtering.Filter;
 4 | import com.zendesk.maxwell.schema.*;
 5 | 
 6 | public class DatabaseCreate extends SchemaChange {
 7 | 	public final String database;
 8 | 	private final boolean ifNotExists;
 9 | 	public final String charset;
10 | 
11 | 	public DatabaseCreate(String database, boolean ifNotExists, String charset) {
12 | 		this.database = database;
13 | 		this.ifNotExists = ifNotExists;
14 | 		this.charset = charset;
15 | 	}
16 | 
17 | 	@Override
18 | 	public ResolvedDatabaseCreate resolve(Schema schema) throws InvalidSchemaError {
19 | 		if ( ifNotExists && schema.hasDatabase(database) )
20 | 			return null;
21 | 
22 | 		String chset;
23 | 		if ( this.charset == null )
24 | 			chset = schema.getCharset();
25 | 		else
26 | 			chset = this.charset;
27 | 
28 | 		return new ResolvedDatabaseCreate(database, chset);
29 | 	}
30 | 
31 | 	@Override
32 | 	public boolean isBlacklisted(Filter filter) {
33 | 		if ( filter == null ) {
34 | 			return false;
35 | 		} else {
36 | 			return filter.isDatabaseBlacklisted(database);
37 | 		}
38 | 	}
39 | 
40 | }
41 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/DatabaseDrop.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.filtering.Filter;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | 
 6 | public class DatabaseDrop extends SchemaChange {
 7 | 	public String database;
 8 | 	public boolean ifExists;
 9 | 
10 | 	public DatabaseDrop(String database, boolean ifExists) {
11 | 		this.database = database;
12 | 		this.ifExists = ifExists;
13 | 	}
14 | 
15 | 	@Override
16 | 	public ResolvedDatabaseDrop resolve(Schema schema) throws InvalidSchemaError {
17 | 		if ( ifExists && !schema.hasDatabase(database) )
18 | 			return null;
19 | 
20 | 		return new ResolvedDatabaseDrop(this.database);
21 | 	}
22 | 
23 | 	@Override
24 | 	public boolean isBlacklisted(Filter filter) {
25 | 		if ( filter == null ) {
26 | 			return false;
27 | 		} else {
28 | 			return filter.isDatabaseBlacklisted(database);
29 | 		}
30 | 	}
31 | 
32 | }
33 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/DeferredPositionUpdate.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | public class DeferredPositionUpdate {
 4 | 	public String column;
 5 | 	public ColumnPosition position;
 6 | 
 7 | 	public DeferredPositionUpdate(String column, ColumnPosition position) {
 8 | 		this.column = column;
 9 | 		this.position = position;
10 | 	}
11 | }
12 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/InvalidSchemaError.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.schema.ddl;
2 | 
3 | public class InvalidSchemaError extends Exception {
4 | 	public InvalidSchemaError (String message) { super(message); }
5 | 	private static final long serialVersionUID = 1L;
6 | }
7 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/MaxwellSQLSyntaxError.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | class MaxwellSQLSyntaxError extends RuntimeException {
 4 | 	private static final long serialVersionUID = 140545518818187219L;
 5 | 
 6 | 	public MaxwellSQLSyntaxError(String message) {
 7 | 		super(message);
 8 | 	}
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/RemoveColumnMod.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | class RemoveColumnMod extends ColumnMod {
 8 | 	private final boolean ifExists;
 9 | 	public RemoveColumnMod(String name, boolean ifExists) {
10 | 		super(name);
11 | 		this.ifExists = ifExists;
12 | 	}
13 | 
14 | 	@Override
15 | 	public void apply(Table table, List<DeferredPositionUpdate> deferred) throws InvalidSchemaError {
16 | 		int originalIndex = table.findColumnIndex(name);
17 | 
18 | 		if ( originalIndex == -1 ) {
19 | 			if ( ifExists )
20 | 				return;
21 | 			else
22 | 				throw new InvalidSchemaError("Could not find column " + name + " in " + table.getName());
23 | 		}
24 | 
25 | 		table.removeColumn(originalIndex);
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/RenameColumnMod.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Table;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | public class RenameColumnMod extends ColumnMod  {
 8 | 	private final String oldName, newName;
 9 | 
10 | 	public RenameColumnMod(String oldName, String newName) {
11 | 		super(oldName);
12 | 		this.oldName = oldName;
13 | 		this.newName = newName;
14 | 	}
15 | 
16 | 	@Override
17 | 	public void apply(Table table, List<DeferredPositionUpdate> deferred) throws InvalidSchemaError {
18 | 		int idx = originalIndex(table);
19 | 		table.renameColumn(idx, newName);
20 | 	}
21 | }
22 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ReparseSQLException.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import org.antlr.v4.runtime.misc.ParseCancellationException;
 4 | 
 5 | public class ReparseSQLException extends ParseCancellationException {
 6 | 	private final String sql;
 7 | 
 8 | 	public ReparseSQLException(String sql) {
 9 | 		this.sql = sql;
10 | 	}
11 | 
12 | 	public String getSQL() {
13 | 		return this.sql;
14 | 	}
15 | }
16 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedDatabaseAlter.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Database;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | 
 6 | public class ResolvedDatabaseAlter extends ResolvedSchemaChange {
 7 | 	public String database;
 8 | 	public String charset;
 9 | 
10 | 	public ResolvedDatabaseAlter() {}
11 | 	public ResolvedDatabaseAlter(String database, String charset) {
12 | 		this.database = database;
13 | 		this.charset = charset;
14 | 	}
15 | 
16 | 	@Override
17 | 	public void apply(Schema schema) throws InvalidSchemaError {
18 | 		if ( charset == null )
19 | 			return;
20 | 
21 | 		Database d = schema.findDatabaseOrThrow(database);
22 | 
23 | 		if ( !d.getCharset().equals(charset) )
24 | 			d.setCharset(charset);
25 | 	}
26 | 
27 | 	@Override
28 | 	public String databaseName() {
29 | 		return database;
30 | 	}
31 | 
32 | 	@Override
33 | 	public String tableName() {
34 | 		return null;
35 | 	}
36 | }
37 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedDatabaseCreate.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Database;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | 
 6 | public class ResolvedDatabaseCreate extends ResolvedSchemaChange {
 7 | 	public String database;
 8 | 	public String charset;
 9 | 
10 | 	public ResolvedDatabaseCreate() { }
11 | 	public ResolvedDatabaseCreate(String database, String charset) {
12 | 		this.database = database;
13 | 		this.charset = charset;
14 | 	}
15 | 
16 | 	@Override
17 | 	public void apply(Schema schema) throws InvalidSchemaError {
18 | 		if ( schema.hasDatabase(database) )
19 | 			throw new InvalidSchemaError("Unexpectedly asked to create existing database " + database);
20 | 
21 | 		schema.addDatabase(new Database(database, charset));
22 | 	}
23 | 
24 | 	@Override
25 | 	public String databaseName() {
26 | 		return database;
27 | 	}
28 | 
29 | 	@Override
30 | 	public String tableName() {
31 | 		return null;
32 | 	}
33 | }
34 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedDatabaseDrop.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Database;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | 
 6 | public class ResolvedDatabaseDrop extends ResolvedSchemaChange {
 7 | 	public String database;
 8 | 
 9 | 	public ResolvedDatabaseDrop() { }
10 | 	public ResolvedDatabaseDrop(String database) {
11 | 		this.database = database;
12 | 	}
13 | 
14 | 	@Override
15 | 	public void apply(Schema schema) throws InvalidSchemaError {
16 | 		Database d = schema.findDatabaseOrThrow(database);
17 | 		schema.removeDatabase(d);
18 | 	}
19 | 
20 | 	@Override
21 | 	public String databaseName() {
22 | 		return database;
23 | 	}
24 | 
25 | 	@Override
26 | 	public String tableName() {
27 | 		return null;
28 | 	}
29 | }
30 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedSchemaChange.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.fasterxml.jackson.annotation.JsonSubTypes;
 4 | import com.fasterxml.jackson.annotation.JsonTypeInfo;
 5 | import com.zendesk.maxwell.filtering.Filter;
 6 | import com.zendesk.maxwell.schema.Schema;
 7 | 
 8 | @JsonTypeInfo(use=JsonTypeInfo.Id.NAME, include=JsonTypeInfo.As.PROPERTY, property="type")
 9 | @JsonSubTypes({
10 | 		@JsonSubTypes.Type(value = ResolvedTableAlter.class, name = "table-alter"),
11 | 		@JsonSubTypes.Type(value = ResolvedTableCreate.class, name = "table-create"),
12 | 		@JsonSubTypes.Type(value = ResolvedTableDrop.class, name = "table-drop"),
13 | 		@JsonSubTypes.Type(value = ResolvedDatabaseAlter.class, name = "database-alter"),
14 | 		@JsonSubTypes.Type(value = ResolvedDatabaseCreate.class, name = "database-create"),
15 | 		@JsonSubTypes.Type(value = ResolvedDatabaseDrop.class, name = "database-drop"),
16 | })
17 | 
18 | public abstract class ResolvedSchemaChange {
19 | 	public abstract void apply(Schema originalSchema) throws InvalidSchemaError;
20 | 
21 | 	public abstract String databaseName();
22 | 
23 | 	public abstract String tableName();
24 | 
25 | 	public boolean shouldOutput(Filter filter) {
26 | 		String table = tableName();
27 | 		if ( table == null )
28 | 			table = "";
29 | 
30 | 		return Filter.includes(filter, databaseName(), table);
31 | 	};
32 | }
33 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedTableAlter.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.fasterxml.jackson.annotation.JsonProperty;
 4 | import com.zendesk.maxwell.filtering.Filter;
 5 | import com.zendesk.maxwell.schema.Database;
 6 | import com.zendesk.maxwell.schema.Schema;
 7 | import com.zendesk.maxwell.schema.Table;
 8 | 
 9 | public class ResolvedTableAlter extends ResolvedSchemaChange {
10 | 	public String database;
11 | 	public String table;
12 | 
13 | 	@JsonProperty("old")
14 | 	public Table oldTable;
15 | 
16 | 	@JsonProperty("def")
17 | 	public Table newTable;
18 | 
19 | 	public ResolvedTableAlter() { }
20 | 	public ResolvedTableAlter(String database, String table, Table oldTable, Table newTable) {
21 | 		this();
22 | 		this.database = database;
23 | 		this.table = table;
24 | 		this.oldTable = oldTable;
25 | 		this.newTable = newTable;
26 | 	}
27 | 
28 | 	@Override
29 | 	public void apply(Schema schema) throws InvalidSchemaError {
30 | 		Database oldDatabase = schema.findDatabaseOrThrow(this.database);
31 | 		Table table = oldDatabase.findTableOrThrow(this.table);
32 | 
33 | 		Database newDatabase;
34 | 		if ( this.database.equals(newTable.database) )
35 | 			newDatabase = oldDatabase;
36 | 		else
37 | 			newDatabase = schema.findDatabaseOrThrow(newTable.database);
38 | 
39 | 		oldDatabase.removeTable(this.table);
40 | 		newDatabase.addTable(newTable);
41 | 	}
42 | 
43 | 	@Override
44 | 	public String databaseName() {
45 | 		return database;
46 | 	}
47 | 
48 | 	@Override
49 | 	public String tableName() {
50 | 		return table;
51 | 	}
52 | 
53 | 	@Override
54 | 	public boolean shouldOutput(Filter filter) {
55 | 		return Filter.includes(filter, database, oldTable.getName()) &&
56 | 			Filter.includes(filter, database, newTable.getName());
57 | 	}
58 | }
59 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedTableCreate.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Database;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | import com.zendesk.maxwell.schema.Table;
 6 | 
 7 | public class ResolvedTableCreate extends ResolvedSchemaChange {
 8 | 	public String database;
 9 | 	public String table;
10 | 	public Table def;
11 | 
12 | 
13 | 	public ResolvedTableCreate() {}
14 | 	public ResolvedTableCreate(Table t) {
15 | 		this.database = t.database;
16 | 		this.table = t.name;
17 | 		this.def = t;
18 | 	}
19 | 
20 | 	@Override
21 | 	public void apply(Schema schema) throws InvalidSchemaError {
22 | 		Database d = schema.findDatabaseOrThrow(this.database);
23 | 
24 | 		if ( d.hasTable(this.table) )
25 | 			throw new InvalidSchemaError("Unexpectedly asked to create existing table " + this.table);
26 | 
27 | 		d.addTable(this.def);
28 | 	}
29 | 
30 | 	@Override
31 | 	public String databaseName() {
32 | 		return database;
33 | 	}
34 | 
35 | 	@Override
36 | 	public String tableName() {
37 | 		return table;
38 | 	}
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/ResolvedTableDrop.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.schema.Database;
 4 | import com.zendesk.maxwell.schema.Schema;
 5 | 
 6 | public class ResolvedTableDrop extends ResolvedSchemaChange {
 7 | 	public String database;
 8 | 	public String table;
 9 | 
10 | 	public ResolvedTableDrop() { }
11 | 	public ResolvedTableDrop(String database, String table) {
12 | 		this.database = database;
13 | 		this.table = table;
14 | 	}
15 | 
16 | 	@Override
17 | 	public void apply(Schema schema) throws InvalidSchemaError {
18 | 		Database d = schema.findDatabaseOrThrow(this.database);
19 | 		d.findTableOrThrow(this.table);
20 | 
21 | 		d.removeTable(this.table);
22 | 	}
23 | 
24 | 	@Override
25 | 	public String databaseName() {
26 | 		return database;
27 | 	}
28 | 
29 | 	@Override
30 | 	public String tableName() {
31 | 		return table;
32 | 	}
33 | }
34 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/TableDrop.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema.ddl;
 2 | 
 3 | import com.zendesk.maxwell.filtering.Filter;
 4 | import com.zendesk.maxwell.schema.*;
 5 | 
 6 | public class TableDrop extends SchemaChange {
 7 | 	public String database;
 8 | 	final String table;
 9 | 	final boolean ifExists;
10 | 
11 | 	public TableDrop(String database, String table, boolean ifExists) {
12 | 		this.database = database;
13 | 		this.table = table;
14 | 		this.ifExists = ifExists;
15 | 	}
16 | 
17 | 	@Override
18 | 	public ResolvedTableDrop resolve(Schema schema) {
19 | 		if ( ifExists ) {
20 | 			Database d = schema.findDatabase(this.database);
21 | 			if ( d == null || !d.hasTable(table) )
22 | 				return null;
23 | 		}
24 | 
25 | 		return new ResolvedTableDrop(database, table);
26 | 	}
27 | 
28 | 	@Override
29 | 	public boolean isBlacklisted(Filter filter) {
30 | 		if ( filter == null ) {
31 | 			return false;
32 | 		} else {
33 | 			return filter.isTableBlacklisted(this.database, this.table);
34 | 		}
35 | 	}
36 | 
37 | }
38 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/ddl/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * the mysql DDL schema parser lives here.
3 |  */
4 | package com.zendesk.maxwell.schema.ddl;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/schema/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * stores and updates (via SQL parsing) a view of the mysql schema
3 |  */
4 | package com.zendesk.maxwell.schema;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/scripting/WrappedDDLMap.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.scripting;
 2 | 
 3 | import com.zendesk.maxwell.replication.BinlogPosition;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | import com.zendesk.maxwell.schema.ddl.DDLMap;
 6 | 
 7 | import java.util.LinkedHashMap;
 8 | import java.util.Map;
 9 | 
10 | // we pass this little wrapper into the javascript interface.
11 | // this class is here so that we're not exposing the full RowMap,
12 | // and so we can snake_case things properly.
13 | public class WrappedDDLMap {
14 | 	private final DDLMap row;
15 | 
16 | 	public WrappedDDLMap(DDLMap row) {
17 | 		this.row = row;
18 | 	}
19 | 
20 | 	public LinkedHashMap<String, Object> getExtra_attributes() {
21 | 		return row.getExtraAttributes();
22 | 	}
23 | 
24 | 	public String getType() {
25 | 		return row.getRowType();
26 | 	}
27 | 
28 | 	public String getTable() {
29 | 		return row.getTable();
30 | 	}
31 | 
32 | 	public String getDatabase() {
33 | 		return row.getDatabase();
34 | 	}
35 | 
36 | 	public String getSql() {
37 | 		return row.getSql();
38 | 	}
39 | 
40 | 	public Map<String, Object> getChange() {
41 | 		return row.getChangeMap();
42 | 	}
43 | 
44 | 	public String getPosition() {
45 | 		BinlogPosition p = row.getPosition().getBinlogPosition();
46 | 
47 | 		if ( p == null )
48 | 			return null;
49 | 
50 | 		if ( p.getGtid() != null )
51 | 			return p.getGtid();
52 | 		else
53 | 			return p.getFile() + ":" + p.getOffset();
54 | 	}
55 | 
56 | 	public Long getTimestamp() {
57 | 		return row.getTimestamp();
58 | 	}
59 | 
60 | 	public void suppress() {
61 | 		row.suppress();
62 | 	}
63 | 
64 | 	public void setKafka_topic(String topic) {
65 | 		row.setKafkaTopic(topic);
66 | 	}
67 | }
68 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/scripting/WrappedHeartbeatMap.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.scripting;
 2 | 
 3 | import com.zendesk.maxwell.replication.BinlogPosition;
 4 | import com.zendesk.maxwell.row.HeartbeatRowMap;
 5 | import com.zendesk.maxwell.row.RowMap;
 6 | import com.zendesk.maxwell.schema.ddl.DDLMap;
 7 | 
 8 | import java.util.LinkedHashMap;
 9 | import java.util.Map;
10 | 
11 | // we pass this little wrapper into the javascript interface.
12 | // this class is here so that we're not exposing the full RowMap,
13 | // and so we can snake_case things properly.
14 | public class WrappedHeartbeatMap {
15 | 	private final HeartbeatRowMap row;
16 | 
17 | 	public WrappedHeartbeatMap(HeartbeatRowMap row) {
18 | 		this.row = row;
19 | 	}
20 | 
21 | 	public String getPosition() {
22 | 		BinlogPosition p = row.getPosition().getBinlogPosition();
23 | 
24 | 		if ( p == null )
25 | 			return null;
26 | 
27 | 		if ( p.getGtid() != null )
28 | 			return p.getGtid();
29 | 		else
30 | 			return p.getFile() + ":" + p.getOffset();
31 | 	}
32 | 
33 | 	public Long getTimestamp() {
34 | 		return row.getTimestamp();
35 | 	}
36 | 
37 | 	public Long getHeartbeat() {
38 | 		return row.getPosition().getLastHeartbeatRead();
39 | 	}
40 | }
41 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/scripting/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * interfaces and code for javascript filters
3 |  */
4 | package com.zendesk.maxwell.scripting;
5 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/ConnectionPool.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import com.zendesk.maxwell.errors.DuplicateProcessException;
 4 | 
 5 | import java.sql.Connection;
 6 | import java.sql.SQLException;
 7 | import java.util.NoSuchElementException;
 8 | 
 9 | public interface ConnectionPool {
10 | 	@FunctionalInterface
11 | 	public interface RetryableSQLFunction<T> {
12 | 		void apply(T t) throws SQLException, NoSuchElementException, DuplicateProcessException;
13 | 	}
14 | 
15 | 	Connection getConnection() throws SQLException;
16 | 	void release();
17 | 
18 | 	void probe() throws SQLException;
19 | 	void withSQLRetry(int nTries, RetryableSQLFunction<Connection> inner)
20 | 		throws SQLException, NoSuchElementException, DuplicateProcessException;
21 | }
22 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/DynamicEnum.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import java.util.HashMap;
 4 | 
 5 | public class DynamicEnum {
 6 | 	private long counter = 0;
 7 | 	private final long maxLength;
 8 | 	private final HashMap<Long, String> forwardMap = new HashMap<>();
 9 | 	private final HashMap<String, Long> backwardsMap = new HashMap<>();
10 | 
11 | 	public DynamicEnum(long maxLength) {
12 | 		this.maxLength = maxLength;
13 | 	}
14 | 
15 | 	public String get(byte byID) {
16 | 		return get((long) byID);
17 | 	}
18 | 
19 | 	public String get(short byID) {
20 | 		return get((long) byID);
21 | 	}
22 | 
23 | 	public String get(int byID) {
24 | 		return get((long) byID);
25 | 	}
26 | 
27 | 	public String get(long byID) {
28 | 		return forwardMap.get(byID);
29 | 	}
30 | 
31 | 	public synchronized long get(String byString) {
32 | 		if ( byString == null )
33 | 			return -1;
34 | 
35 | 		Long id = backwardsMap.get(byString);
36 | 		if ( id == null ) {
37 | 			forwardMap.put(counter, byString);
38 | 			backwardsMap.put(byString, counter);
39 | 			id = counter;
40 | 			if ( ++counter >= maxLength )
41 | 				throw new RuntimeException("Overflowed DynamicEnum!");
42 | 		}
43 | 		return id;
44 | 	}
45 | }
46 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/Logging.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import org.apache.logging.log4j.Level;
 4 | import org.apache.logging.log4j.LogManager;
 5 | import org.apache.logging.log4j.core.LoggerContext;
 6 | import org.apache.logging.log4j.core.config.Configuration;
 7 | import org.apache.logging.log4j.core.config.LoggerConfig;
 8 | import org.slf4j.bridge.SLF4JBridgeHandler;
 9 | 
10 | public class Logging {
11 | 	public static void setLevel(String level) {
12 | 		LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
13 | 		Configuration config = ctx.getConfiguration();
14 | 		LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
15 | 		loggerConfig.setLevel(Level.valueOf(level));
16 | 		ctx.updateLoggers();  // This causes all Loggers to refetch information from their LoggerConfig.
17 | 	}
18 | 
19 | 	public static void setupLogBridging() {
20 | 		// Optionally remove existing handlers attached to j.u.l root logger
21 | 		SLF4JBridgeHandler.removeHandlersForRootLogger();  // (since SLF4J 1.6.5)
22 | 
23 | 		// add SLF4JBridgeHandler to j.u.l's root logger, should be done once during
24 | 		// the initialization phase of your application
25 | 		SLF4JBridgeHandler.install();
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/MaxwellOptionParser.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import joptsimple.OptionParser;
 4 | import joptsimple.OptionSpecBuilder;
 5 | 
 6 | import java.io.IOException;
 7 | import java.io.PrintStream;
 8 | import java.util.ArrayList;
 9 | import java.util.HashMap;
10 | 
11 | public class MaxwellOptionParser extends OptionParser {
12 | 	private String currentSection = null;
13 | 	private ArrayList<String> sectionNames = new ArrayList<>();
14 | 	private HashMap<String, ArrayList<String>> sections = new HashMap<>();
15 | 	private MaxwellHelpFormatter helpFormatter = new MaxwellHelpFormatter(200, 4, sections, sectionNames);
16 | 
17 | 	public MaxwellOptionParser() {
18 | 		this.formatHelpWith(helpFormatter);
19 | 	}
20 | 
21 | 	public void addToSection(String optionName)  {
22 | 		ArrayList<String> list = sections.computeIfAbsent(currentSection, k -> new ArrayList<>());
23 | 		list.add(optionName);
24 | 	}
25 | 
26 | 	@Override
27 | 	public OptionSpecBuilder accepts(String option, String description) {
28 | 		addToSection(option);
29 | 		return super.accepts(option, description);
30 | 	}
31 | 
32 | 	private int separatorIndex = 0;
33 | 	public void separator() {
34 | 		this.accepts("__separator_" + ++separatorIndex, "");
35 | 	}
36 | 
37 | 	public void section(String section) {
38 | 		// this puts in a separator not in a section.
39 | 		this.accepts("__section_" + section);
40 | 
41 | 		this.currentSection = section;
42 | 		this.sectionNames.add(section);
43 | 	}
44 | 
45 | 	public void printHelpOn(PrintStream err, String section) throws IOException {
46 | 		this.helpFormatter.setSection(section);
47 | 		super.printHelpOn(err);
48 | 	}
49 | }
50 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/RunLoopProcess.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import java.util.concurrent.TimeoutException;
 4 | 
 5 | abstract public class RunLoopProcess implements StoppableTask {
 6 | 	protected volatile StoppableTaskState taskState;
 7 | 	private Thread thread;
 8 | 
 9 | 	public RunLoopProcess() {
10 | 		this.taskState = new StoppableTaskState(this.getClass().getName());
11 | 	}
12 | 
13 | 	public void requestStop() {
14 | 		this.taskState.requestStop();
15 | 
16 | 		interrupt();
17 | 	}
18 | 
19 | 	public void interrupt() {
20 | 		if ( this.thread != null )
21 | 			this.thread.interrupt();
22 | 	}
23 | 
24 | 	public void awaitStop(Long timeout) throws TimeoutException {
25 | 		this.taskState.awaitStop(thread, timeout);
26 | 	}
27 | 
28 | 	public void runLoop() throws Exception {
29 | 		this.thread = Thread.currentThread();
30 | 		this.beforeStart();
31 | 
32 | 		try {
33 | 			while (this.taskState.isRunning()) {
34 | 				work();
35 | 			}
36 | 		} finally {
37 | 			this.beforeStop();
38 | 			this.taskState.stopped();
39 | 		}
40 | 	}
41 | 
42 | 	protected abstract void work() throws Exception;
43 | 	protected void beforeStart() throws Exception { }
44 | 	protected void beforeStop() throws Exception { }
45 | }
46 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/RunState.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.util;
2 | 
3 | public enum RunState { STOPPED, RUNNING, REQUEST_STOP }
4 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/Sql.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import org.apache.commons.lang3.StringUtils;
 4 | 
 5 | import java.sql.PreparedStatement;
 6 | import java.sql.SQLException;
 7 | 
 8 | public class Sql {
 9 | 	public static String inListSQL(int count) {
10 | 		return "(" + StringUtils.repeat("?", ", ", count) + ")";
11 | 	}
12 | 
13 | 	public static void prepareInList(PreparedStatement s, int offset, Iterable<?> list) throws SQLException {
14 | 		for ( Object o : list ) {
15 | 			s.setObject(offset++, o);
16 | 		}
17 | 	}
18 | }
19 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/StoppableTask.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell.util;
2 | 
3 | import java.util.concurrent.TimeoutException;
4 | 
5 | public interface StoppableTask {
6 | 	void requestStop() throws Exception;
7 | 	void awaitStop(Long timeout) throws TimeoutException;
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/StoppableTaskState.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.util;
 2 | 
 3 | import org.slf4j.Logger;
 4 | import org.slf4j.LoggerFactory;
 5 | 
 6 | import java.util.concurrent.TimeoutException;
 7 | 
 8 | public class StoppableTaskState {
 9 | 	private static Logger LOGGER = LoggerFactory.getLogger(StoppableTaskState.class);
10 | 	private volatile RunState state;
11 | 	private final String description;
12 | 
13 | 	public StoppableTaskState(String description) {
14 | 		state = RunState.RUNNING;
15 | 		this.description = description;
16 | 	}
17 | 
18 | 	public boolean isRunning() {
19 | 		return state == RunState.RUNNING;
20 | 	}
21 | 
22 | 	public synchronized void requestStop() {
23 | 		LOGGER.debug("{} requestStop() called (in state: {})", description, state);
24 | 		if (isRunning()) {
25 | 			this.state = RunState.REQUEST_STOP;
26 | 		}
27 | 	}
28 | 
29 | 	public void stopped() {
30 | 		this.state = RunState.STOPPED;
31 | 	}
32 | 
33 | 	public synchronized void awaitStop(Thread t, long timeoutMS) throws TimeoutException {
34 | 		/* foot tapping */
35 | 		for (long left = timeoutMS; left > 0 && this.state == RunState.REQUEST_STOP; left -= 10)
36 | 			try { Thread.sleep(10); } catch (InterruptedException e) { }
37 | 
38 | 		/* very impatient throat clear */
39 | 		if (t != null) {
40 | 			t.interrupt();
41 | 		}
42 | 
43 | 		try { Thread.sleep(100); } catch (InterruptedException e) { }
44 | 
45 | 		if( this.state != RunState.STOPPED ) {
46 | 			throw new TimeoutException(
47 | 				"Timed out trying waiting for " + description + " process to stop after " + timeoutMS + "ms."
48 | 			);
49 | 		}
50 | 	}
51 | 
52 | 	public RunState getState() {
53 | 		return state;
54 | 	}
55 | }
56 | 


--------------------------------------------------------------------------------
/src/main/java/com/zendesk/maxwell/util/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 |  * grab-bag of code
3 |  */
4 | package com.zendesk.maxwell.util;
5 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j2.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <Configuration status="WARN">
 3 |   <Appenders>
 4 |     <Console name="Console" target="SYSTEM_ERR">
 5 |       <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5level %logger{1} - %msg%n"/>
 6 |     </Console>
 7 |   </Appenders>
 8 |   <Loggers>
 9 |     <Root level="info">
10 |       <AppenderRef ref="Console"/>
11 |     </Root>
12 |    <Logger name="com.mchange.v2" level="error" additivity="false">
13 |       <AppenderRef ref="Console"/>
14 |     </Logger>
15 |   </Loggers>
16 | </Configuration>
17 | 


--------------------------------------------------------------------------------
/src/main/resources/sql/maxwell_schema_bootstrap.sql:
--------------------------------------------------------------------------------
 1 | CREATE TABLE IF NOT EXISTS `bootstrap` (
 2 |   id              bigint auto_increment NOT NULL primary key,
 3 |   database_name   varchar(255) charset 'utf8' NOT NULL,
 4 |   table_name      varchar(255) charset 'utf8' NOT NULL,
 5 |   where_clause    text default NULL,
 6 |   is_complete     tinyint(1) unsigned NOT NULL default 0,
 7 |   inserted_rows   bigint(20) unsigned NOT NULL DEFAULT 0,
 8 |   total_rows      bigint(20) unsigned NOT NULL DEFAULT 0,
 9 |   created_at      DATETIME default NULL,
10 |   started_at      DATETIME default NULL,
11 |   completed_at    DATETIME default NULL,
12 |   binlog_file     varchar(255) default NULL,
13 |   binlog_position int unsigned default 0,
14 |   client_id       varchar(255) charset 'latin1' not null default 'maxwell',
15 |   comment         varchar(255) charset 'utf8' default NULL
16 | );
17 | 


--------------------------------------------------------------------------------
/src/main/resources/sql/maxwell_schema_heartbeats.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE IF NOT EXISTS `heartbeats` (
2 |   server_id int unsigned not null,
3 |   client_id varchar(255) charset latin1 not null default 'maxwell',
4 |   heartbeat bigint not null,
5 |   primary key(server_id, client_id)
6 | );
7 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/JUnitNameFilterFactory.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | import org.junit.runner.Description;
 4 | import org.junit.runner.FilterFactory;
 5 | import org.junit.runner.FilterFactoryParams;
 6 | import org.junit.runner.manipulation.Filter;
 7 | 
 8 | /**
 9 |  * Created by ben on 12/26/16.
10 |  */
11 | public class JUnitNameFilterFactory implements FilterFactory {
12 | 	private class JUnitNameFilter extends Filter {
13 | 		private final String filterString;
14 | 
15 | 		private JUnitNameFilter(String filterString) {
16 | 			this.filterString = filterString;
17 | 		}
18 | 		@Override
19 | 		public boolean shouldRun(Description description) {
20 | 			String methodName = description.getMethodName();
21 | 			if ( methodName == null )
22 | 				return true;
23 | 			return filterString.equalsIgnoreCase(methodName);
24 | 		}
25 | 
26 | 		@Override
27 | 		public String describe() {
28 | 			return "name = " + filterString;
29 | 		}
30 | 	}
31 | 	@Override
32 | 	public Filter createFilter(FilterFactoryParams params) throws FilterNotCreatedException {
33 | 		return new JUnitNameFilter(params.getArgs().toString());
34 | 	}
35 | }
36 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/MaxwellTestSupportCallback.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | import java.sql.SQLException;
 4 | 
 5 | public class MaxwellTestSupportCallback {
 6 | 	public void beforeReplicatorStart(MysqlIsolatedServer mysql) throws SQLException {}
 7 | 	public void afterReplicatorStart(MysqlIsolatedServer mysql) throws SQLException {}
 8 | 	public void beforeTerminate(MysqlIsolatedServer mysql) throws Exception { }
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/Mysql57TestSuite.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | import com.zendesk.maxwell.schema.SchemaCaptureTest;
 4 | import com.zendesk.maxwell.schema.ddl.DDLIntegrationTest;
 5 | import org.junit.experimental.categories.Categories;
 6 | import org.junit.runner.RunWith;
 7 | import org.junit.runners.Suite;
 8 | 
 9 | @RunWith(Categories.class)
10 | @Categories.IncludeCategory(Mysql57Tests.class)
11 | @Suite.SuiteClasses({
12 | 	BootstrapIntegrationTest.class,
13 | 	SchemaCaptureTest.class,
14 | 	MaxwellIntegrationTest.class,
15 | 	DDLIntegrationTest.class,
16 | 	EmbeddedMaxwellTest.class
17 | })
18 | public class Mysql57TestSuite {}
19 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/Mysql57Tests.java:
--------------------------------------------------------------------------------
1 | package com.zendesk.maxwell;
2 | 
3 | public class Mysql57Tests { }
4 | 
5 | 
6 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/TestWithNameLogging.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell;
 2 | 
 3 | import org.junit.Rule;
 4 | import org.junit.rules.TestWatcher;
 5 | import org.junit.runner.Description;
 6 | import org.slf4j.Logger;
 7 | import org.slf4j.LoggerFactory;
 8 | 
 9 | /**
10 |  * Created by ben on 8/31/16.
11 |  */
12 | public class TestWithNameLogging {
13 | 	private static Logger logger = LoggerFactory.getLogger("TestHarness");
14 | 
15 | 	@Rule
16 | 	public TestWatcher watchman = new TestWatcher() {
17 | 		@Override
18 | 		public void starting(final Description method) {
19 | 			logger.info("========== BEGIN: " + method.getMethodName());
20 | 		}
21 | 
22 | 		@Override
23 | 		protected void finished(Description method) {
24 | 			logger.info("========== END: " + method.getMethodName());
25 | 		}
26 | 	};
27 | }
28 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/benchmark/BenchmarkProducer.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.benchmark;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.producer.AbstractProfilingProducer;
 5 | import com.zendesk.maxwell.row.RowMap;
 6 | 
 7 | public class BenchmarkProducer extends AbstractProfilingProducer {
 8 | 	private long lastRowReceivedAt = 0;
 9 | 	private long rowsDiscarded = 0, discardFirstRows;
10 | 
11 | 	public BenchmarkProducer(MaxwellContext context, long discardFirstRows) {
12 | 		super(context);
13 | 
14 | 		this.discardFirstRows = discardFirstRows;
15 | 		new Thread(() -> {
16 | 			while (true) {
17 | 				checkDone();
18 | 				try {
19 | 					Thread.sleep(100);
20 | 				} catch (InterruptedException e) { }
21 | 			}
22 | 
23 | 		}).start();
24 | 	}
25 | 
26 | 	// crude, but should work - wait until there's no rows coming in and log.
27 | 	private void checkDone() {
28 | 		String msg;
29 | 		long now = System.currentTimeMillis();
30 | 		if ( lastRowReceivedAt > 0 && now - lastRowReceivedAt > 2000 )	 {
31 | 			double seconds = (lastRowReceivedAt - this.startTime) / 1000.0d;
32 | 			msg = String.format("Producer received %d rows in %.2f seconds", this.count, seconds );
33 | 			System.out.println(msg);
34 | 			msg = String.format("%.0f rows per second", this.count / seconds);
35 | 			System.out.println(msg);
36 | 			System.exit(0);
37 | 		}
38 | 	}
39 | 
40 | 	@Override
41 | 	public void push(RowMap r) throws Exception {
42 | 		if ( rowsDiscarded < discardFirstRows ) {
43 | 			String value = r.toJSON();
44 | 			rowsDiscarded++;
45 | 		} else {
46 | 			super.push(r);
47 | 			this.lastRowReceivedAt = System.currentTimeMillis();
48 | 		}
49 | 	}
50 | }
51 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/benchmark/BenchmarkProducerFactory.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.benchmark;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellContext;
 4 | import com.zendesk.maxwell.producer.AbstractProducer;
 5 | import com.zendesk.maxwell.producer.ProducerFactory;
 6 | 
 7 | public class BenchmarkProducerFactory implements ProducerFactory {
 8 | 	private long skipRows;
 9 | 	public BenchmarkProducerFactory(long skipRows) {
10 | 		this.skipRows = skipRows;
11 | 	}
12 | 	@Override
13 | 	public AbstractProducer createProducer(MaxwellContext context) {
14 | 		return new BenchmarkProducer(context, skipRows);
15 | 	}
16 | }
17 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/monitoring/MaxwellMonitorIntegrationTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.monitoring;
 2 | 
 3 | import com.zendesk.maxwell.*;
 4 | import com.zendesk.maxwell.row.RowMap;
 5 | import org.junit.Assert;
 6 | import org.junit.Test;
 7 | 
 8 | import java.io.InputStream;
 9 | import java.net.HttpURLConnection;
10 | import java.net.MalformedURLException;
11 | import java.net.URL;
12 | import java.util.List;
13 | 
14 | public class MaxwellMonitorIntegrationTest extends MaxwellTestWithIsolatedServer {
15 | 
16 | 	private boolean probeHTTP() {
17 | 		try {
18 | 			// Create a neat value object to hold the URL
19 | 			URL url = new URL("http://localhost:22222/metrics");
20 | 
21 | 			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
22 | 			InputStream responseStream = connection.getInputStream();
23 | 
24 | 			responseStream.read();
25 | 			return true;
26 | 		} catch (Exception e) {
27 | 			return false;
28 | 		}
29 | 
30 | 	}
31 | 	@Test
32 | 	public void testHTTPEndpoint() throws Exception {
33 | 		String[] sql = {
34 | 				"CREATE TABLE `test`.`tingalingaling` ( id int )",
35 | 				"insert into `test`.`tingalingaling` set id = 5"
36 | 		};
37 | 
38 | 
39 | 		MaxwellTestSupportCallback cb = new MaxwellTestSupportCallback() {
40 | 			@Override
41 | 			public void beforeTerminate(MysqlIsolatedServer mysql) throws Exception {
42 | 				for ( int i = 0 ; i < 5; i++ ) {
43 | 					if ( probeHTTP() ) return;
44 | 					try { Thread.sleep(1000); } catch ( InterruptedException e ) {}
45 | 				}
46 | 				Assert.assertFalse("failed to connect to http server after 5 seconds!", false);
47 | 			}
48 | 		};
49 | 		MaxwellTestSupport.getRowsWithReplicator(server, cb, (config) -> {
50 | 			config.httpPort = 22222;
51 | 			config.metricsReportingType = "http";
52 | 		});
53 | 
54 | 	}
55 | }
56 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/producer/DiagnosticCallbackTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | import static org.junit.Assert.assertTrue;
 6 | 
 7 | public class DiagnosticCallbackTest {
 8 | 
 9 | 	@Test
10 | 	public void testCompleteNormally() throws Exception {
11 | 		// Given
12 | 		KafkaProducerDiagnostic.DiagnosticCallback callback = new KafkaProducerDiagnostic.DiagnosticCallback();
13 | 
14 | 		// When
15 | 		callback.onCompletion(null, null);
16 | 
17 | 		// Then
18 | 		callback.latency.get();
19 | 	}
20 | 
21 | 	@Test
22 | 	public void testCompleteExceptionally() {
23 | 		// Given
24 | 		KafkaProducerDiagnostic.DiagnosticCallback callback = new KafkaProducerDiagnostic.DiagnosticCallback();
25 | 
26 | 		// When
27 | 		callback.onCompletion(null, new RuntimeException("blah"));
28 | 
29 | 		// Then
30 | 		assertTrue(callback.latency.isCompletedExceptionally());
31 | 	}
32 | }
33 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/producer/MaxwellKinesisProducerTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellConfig;
 4 | import com.zendesk.maxwell.MaxwellContext;
 5 | import com.zendesk.maxwell.monitoring.NoOpMetrics;
 6 | import com.zendesk.maxwell.replication.BinlogPosition;
 7 | import com.zendesk.maxwell.replication.Position;
 8 | import com.zendesk.maxwell.row.RowMap;
 9 | import org.junit.Test;
10 | 
11 | import java.util.Properties;
12 | import java.util.ArrayList;
13 | 
14 | import static org.mockito.Mockito.mock;
15 | import static org.mockito.Mockito.when;
16 | 
17 | public class MaxwellKinesisProducerTest {
18 | 
19 | 	private static final long TIMESTAMP_MILLISECONDS = 1496712943447L;
20 | 
21 | 	private static final Position POSITION = new Position(new BinlogPosition(1L, "binlog-0001"), 0L);
22 | 
23 | 	@Test
24 | 	public void dealsWithTooLargeRecord() throws Exception {
25 | 		MaxwellContext context = mock(MaxwellContext.class);
26 | 		MaxwellConfig config = new MaxwellConfig();
27 | 		when(context.getConfig()).thenReturn(config);
28 | 		when(context.getMetrics()).thenReturn(new NoOpMetrics());
29 | 		String kinesisStream = "test-stream";
30 | 		MaxwellKinesisProducer producer = new MaxwellKinesisProducer(context, kinesisStream);
31 | 
32 | 		RowMap rowMap = new RowMap("insert", "MyDatabase", "MyTable", TIMESTAMP_MILLISECONDS, new ArrayList<String>(), POSITION);
33 | 		StringBuilder r = new StringBuilder();
34 | 		for (int i = 0; i < 100_000; i++) {
35 | 			r.append("long string");
36 | 		}
37 | 		rowMap.putData("content", r.toString());
38 | 
39 | 		AbstractAsyncProducer.CallbackCompleter cc = mock(AbstractAsyncProducer.CallbackCompleter.class);
40 | 		producer.sendAsync(rowMap, cc);
41 | 		producer.close();
42 | 	}
43 | }
44 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/producer/NatsOptionMatcher.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer;
 2 | 
 3 | import io.nats.client.Options;
 4 | import org.mockito.ArgumentMatcher;
 5 | 
 6 | public class NatsOptionMatcher implements ArgumentMatcher<Options> {
 7 | 
 8 | 	private final Options expected;
 9 | 
10 | 	public NatsOptionMatcher(final Options expected) {
11 | 		this.expected = expected;
12 | 	}
13 | 
14 | 	@Override
15 | 	public boolean matches(Options options) {
16 | 		return this.expected.getServers().equals(options.getServers());
17 | 	}
18 | }
19 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/producer/partitioners/MaxwellKafkaPartitionerTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.producer.partitioners;
 2 | 
 3 | import com.zendesk.maxwell.row.RowMap;
 4 | import com.zendesk.maxwell.schema.ddl.DDLMap;
 5 | import com.zendesk.maxwell.schema.ddl.ResolvedDatabaseAlter;
 6 | import com.zendesk.maxwell.schema.ddl.ResolvedTableDrop;
 7 | import org.junit.Test;
 8 | 
 9 | import static org.junit.Assert.assertEquals;
10 | import static org.junit.Assert.assertThat;
11 | 
12 | import java.util.ArrayList;
13 | 
14 | public class MaxwellKafkaPartitionerTest {
15 | 	@Test
16 | 	public void testRowMapEqualsDDLPartitioning() {
17 | 		RowMap r = new RowMap("insert", "db", "tbl", 0L, new ArrayList<>(), null);
18 | 		ResolvedDatabaseAlter m = new ResolvedDatabaseAlter("db", "utf8");
19 | 		DDLMap d = new DDLMap(m, 0L, "alter-sql", null, null, null);
20 | 
21 | 		MaxwellKafkaPartitioner p = new MaxwellKafkaPartitioner("murmur3", "database", null, null);
22 | 		assertEquals(p.kafkaPartition(r, 15), p.kafkaPartition(d, 15));
23 | 	}
24 | 
25 | 	@Test
26 | 	public void testDDLFallBack() {
27 | 		ResolvedDatabaseAlter m = new ResolvedDatabaseAlter("some_db", "utf8");
28 | 		DDLMap d = new DDLMap(m, 0L, "alter-sql", null, null, null);
29 | 		MaxwellKafkaPartitioner p = new MaxwellKafkaPartitioner("murmur3", "table", null, "database");
30 | 
31 | 		ResolvedTableDrop m2 = new ResolvedTableDrop("some_db", "some_table");
32 | 		DDLMap d2 = new DDLMap(m2, 0L, "alter-sql", null, null, null);
33 | 		MaxwellKafkaPartitioner p2 = new MaxwellKafkaPartitioner("murmur3", "database", null, "database");
34 | 
35 | 		assertEquals(p2.kafkaPartition(d2, 15), p.kafkaPartition(d, 15));
36 | 	}
37 | }
38 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/replication/MysqlVersionTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.replication;
 2 | 
 3 | import com.zendesk.maxwell.TestWithNameLogging;
 4 | import org.junit.Test;
 5 | 
 6 | import static org.junit.Assert.assertFalse;
 7 | import static org.junit.Assert.assertTrue;
 8 | 
 9 | public class MysqlVersionTest extends TestWithNameLogging {
10 | 	@Test
11 | 	public void testVersionComparison() {
12 | 		MysqlVersion version = new MysqlVersion(6, 5);
13 | 		assertTrue(version.atLeast(6, 5));
14 | 		assertTrue(version.atLeast(6, 4));
15 | 		assertTrue(version.atLeast(5, 9));
16 | 
17 | 		assertFalse(version.atLeast(7, 2));
18 | 		assertFalse(version.atLeast(6, 6));
19 | 	}
20 | }
21 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/replication/TableCacheTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.replication;
 2 | 
 3 | import com.zendesk.maxwell.MaxwellTestWithIsolatedServer;
 4 | import com.zendesk.maxwell.filtering.Filter;
 5 | import com.zendesk.maxwell.schema.Schema;
 6 | import com.zendesk.maxwell.schema.SchemaCapturer;
 7 | import org.junit.Test;
 8 | 
 9 | public class TableCacheTest extends MaxwellTestWithIsolatedServer {
10 | 	@Test
11 | 	public void testHaTables() throws Exception {
12 | 		Schema schema = new SchemaCapturer(server.getConnection(), buildContext().getCaseSensitivity()).capture();
13 | 		TableCache cache = new TableCache("maxwell");
14 | 		// ensure we don't crash on not-really-existant alibaba tables
15 | 		cache.processEvent(schema, new Filter(), false,1L, "mysql", "ha_health_check");
16 | 	}
17 | }
18 | 


--------------------------------------------------------------------------------
/src/test/java/com/zendesk/maxwell/schema/MysqlSchemaStoreTest.java:
--------------------------------------------------------------------------------
 1 | package com.zendesk.maxwell.schema;
 2 | 
 3 | import static org.hamcrest.CoreMatchers.containsString;
 4 | import static org.hamcrest.CoreMatchers.is;
 5 | import static org.hamcrest.CoreMatchers.not;
 6 | import static org.hamcrest.CoreMatchers.nullValue;
 7 | import static org.junit.Assert.*;
 8 | import com.zendesk.maxwell.*;
 9 | import java.sql.ResultSet;
10 | import java.util.List;
11 | 
12 | import com.zendesk.maxwell.recovery.RecoveryInfo;
13 | import com.zendesk.maxwell.replication.BinlogPosition;
14 | import com.zendesk.maxwell.errors.DuplicateProcessException;
15 | import com.zendesk.maxwell.replication.Position;
16 | import org.apache.commons.lang.StringUtils;
17 | import org.junit.Test;
18 | 
19 | public class MysqlSchemaStoreTest extends MaxwellTestWithIsolatedServer {
20 |     @Test
21 |     public void testGetSchemaID() throws Exception {
22 |         BinlogPosition bl_pos = new BinlogPosition(0, "mysql.1234");
23 |         Position pos = new Position(bl_pos, 1);
24 |         MysqlSchemaStore schemaStore = new MysqlSchemaStore(buildContext(), pos);
25 |         assertThat(schemaStore.getSchemaID(), is(1L));
26 | 
27 |         String sql = "CREATE DATABASE `testdb`;";
28 |         String db = "testdb";
29 |         BinlogPosition bl_pos2 = new BinlogPosition(1, "mysql.1234");
30 |         Position pos2 = new Position(bl_pos2, 1);
31 |         schemaStore.processSQL(sql, db, pos2);
32 |         assertThat(schemaStore.getSchemaID(), is(2L));
33 |     }
34 | }
35 | 


--------------------------------------------------------------------------------
/src/test/resources/config/env-var-config.properties:
--------------------------------------------------------------------------------
1 | env_config_prefix=FOO_


--------------------------------------------------------------------------------
/src/test/resources/config/producer-factory-config.properties:
--------------------------------------------------------------------------------
1 | custom_producer.factory=com.zendesk.maxwell.MaxwellConfigTest$TestProducerFactory
2 | custom_producer.foo=bar


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-database:
--------------------------------------------------------------------------------
1 | ALTER {DATABASE | SCHEMA} [db_name]
2 |     alter_specification ...
3 | ALTER {DATABASE | SCHEMA} db_name
4 |     UPGRADE DATA DIRECTORY NAME
5 | 
6 | alter_specification:
7 |     [DEFAULT] CHARACTER SET [=] charset_name
8 |   | [DEFAULT] COLLATE [=] collation_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-event:
--------------------------------------------------------------------------------
1 | ALTER
2 |     [DEFINER = { user | CURRENT_USER }]
3 |     EVENT event_name
4 |     [ON SCHEDULE schedule]
5 |     [ON COMPLETION [NOT] PRESERVE]
6 |     [RENAME TO new_event_name]
7 |     [ENABLE | DISABLE | DISABLE ON SLAVE]
8 |     [COMMENT 'string']
9 |     [DO event_body]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-function:
--------------------------------------------------------------------------------
1 | ALTER FUNCTION func_name [characteristic ...]
2 | 
3 | characteristic:
4 |     COMMENT 'string'
5 |   | LANGUAGE SQL
6 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
7 |   | SQL SECURITY { DEFINER | INVOKER }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-instance:
--------------------------------------------------------------------------------
1 | ALTER INSTANCE ROTATE INNODB MASTER KEY


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-procedure:
--------------------------------------------------------------------------------
1 | ALTER PROCEDURE proc_name [characteristic ...]
2 | 
3 | characteristic:
4 |     COMMENT 'string'
5 |   | LANGUAGE SQL
6 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
7 |   | SQL SECURITY { DEFINER | INVOKER }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-server:
--------------------------------------------------------------------------------
1 | ALTER SERVER  server_name
2 |     OPTIONS (option [, option] ...)


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-tablespace:
--------------------------------------------------------------------------------
1 | ALTER TABLESPACE tablespace_name
2 |     {ADD|DROP} DATAFILE 'file_name'
3 |     [INITIAL_SIZE [=] size]
4 |     [WAIT]
5 |     ENGINE [=] engine_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-user:
--------------------------------------------------------------------------------
 1 | ALTER USER [IF EXISTS]
 2 |     user [auth_option] [, user [auth_option]] ...
 3 |     [REQUIRE {NONE | tls_option [[AND] tls_option] ...}]
 4 |     [WITH resource_option [resource_option] ...]
 5 |     [password_option | lock_option] ...
 6 | 
 7 | ALTER USER [IF EXISTS]
 8 |     USER() IDENTIFIED BY 'auth_string'
 9 | 
10 | user:
11 |     (see Section 6.2.3, “Specifying Account Names”)
12 | 
13 | auth_option: {
14 |     IDENTIFIED BY 'auth_string'
15 |   | IDENTIFIED WITH auth_plugin
16 |   | IDENTIFIED WITH auth_plugin BY 'auth_string'
17 |   | IDENTIFIED WITH auth_plugin AS 'hash_string'
18 | }
19 | 
20 | tls_option: {
21 |    SSL
22 |  | X509
23 |  | CIPHER 'cipher'
24 |  | ISSUER 'issuer'
25 |  | SUBJECT 'subject'
26 | }
27 | 
28 | resource_option: {
29 |     MAX_QUERIES_PER_HOUR count
30 |   | MAX_UPDATES_PER_HOUR count
31 |   | MAX_CONNECTIONS_PER_HOUR count
32 |   | MAX_USER_CONNECTIONS count
33 | }
34 | 
35 | password_option: {
36 |     PASSWORD EXPIRE
37 |   | PASSWORD EXPIRE DEFAULT
38 |   | PASSWORD EXPIRE NEVER
39 |   | PASSWORD EXPIRE INTERVAL N DAY
40 | }
41 | 
42 | lock_option: {
43 |     ACCOUNT LOCK
44 |   | ACCOUNT UNLOCK
45 | }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/alter-view:
--------------------------------------------------------------------------------
1 | ALTER
2 |     [ALGORITHM = {UNDEFINED | MERGE | TEMPTABLE}]
3 |     [DEFINER = { user | CURRENT_USER }]
4 |     [SQL SECURITY { DEFINER | INVOKER }]
5 |     VIEW view_name [(column_list)]
6 |     AS select_statement
7 |     [WITH [CASCADED | LOCAL] CHECK OPTION]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/analyze-table:
--------------------------------------------------------------------------------
1 | ANALYZE [NO_WRITE_TO_BINLOG | LOCAL]
2 |     TABLE tbl_name [, tbl_name] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/cache-index:
--------------------------------------------------------------------------------
 1 | CACHE INDEX
 2 |   tbl_index_list [, tbl_index_list] ...
 3 |   [PARTITION (partition_list | ALL)]
 4 |   IN key_cache_name
 5 | 
 6 | tbl_index_list:
 7 |   tbl_name [[INDEX|KEY] (index_name[, index_name] ...)]
 8 | 
 9 | partition_list:
10 |   partition_name[, partition_name][, ...]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/checksum-table:
--------------------------------------------------------------------------------
1 | CHECKSUM TABLE tbl_name [, tbl_name] ... [QUICK | EXTENDED]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-database:
--------------------------------------------------------------------------------
1 | CREATE {DATABASE | SCHEMA} [IF NOT EXISTS] db_name
2 |     [create_specification] ...
3 | 
4 | create_specification:
5 |     [DEFAULT] CHARACTER SET [=] charset_name
6 |   | [DEFAULT] COLLATE [=] collation_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-event:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     EVENT
 4 |     [IF NOT EXISTS]
 5 |     event_name
 6 |     ON SCHEDULE schedule
 7 |     [ON COMPLETION [NOT] PRESERVE]
 8 |     [ENABLE | DISABLE | DISABLE ON SLAVE]
 9 |     [COMMENT 'string']
10 |     DO event_body;
11 | 
12 | schedule:
13 |     AT timestamp [+ INTERVAL interval] ...
14 |   | EVERY interval
15 |     [STARTS timestamp [+ INTERVAL interval] ...]
16 |     [ENDS timestamp [+ INTERVAL interval] ...]
17 | 
18 | interval:
19 |     quantity {YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE |
20 |               WEEK | SECOND | YEAR_MONTH | DAY_HOUR | DAY_MINUTE |
21 |               DAY_SECOND | HOUR_MINUTE | HOUR_SECOND | MINUTE_SECOND}


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-function-udf:
--------------------------------------------------------------------------------
1 | CREATE [AGGREGATE] FUNCTION function_name
2 |     RETURNS {STRING|INTEGER|REAL|DECIMAL}
3 |     SONAME shared_library_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-index:
--------------------------------------------------------------------------------
 1 | CREATE [UNIQUE | FULLTEXT | SPATIAL] INDEX index_name
 2 |     [index_type]
 3 |     ON tbl_name (key_part,...)
 4 |     [index_option]
 5 |     [algorithm_option | lock_option] ...
 6 | 
 7 | key_part:
 8 |     col_name [(length)] [ASC | DESC]
 9 | 
10 | index_option:
11 |     KEY_BLOCK_SIZE [=] value
12 |   | index_type
13 |   | WITH PARSER parser_name
14 |   | COMMENT 'string'
15 | 
16 | index_type:
17 |     USING {BTREE | HASH}
18 | 
19 | algorithm_option:
20 |     ALGORITHM [=] {DEFAULT | INPLACE | COPY}
21 | 
22 | lock_option:
23 |     LOCK [=] {DEFAULT | NONE | SHARED | EXCLUSIVE}


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-procedure:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     PROCEDURE sp_name ([proc_parameter[,...]])
 4 |     [characteristic ...] routine_body
 5 | 
 6 | CREATE
 7 |     [DEFINER = { user | CURRENT_USER }]
 8 |     FUNCTION sp_name ([func_parameter[,...]])
 9 |     RETURNS type
10 |     [characteristic ...] routine_body
11 | 
12 | proc_parameter:
13 |     [ IN | OUT | INOUT ] param_name type
14 | 
15 | func_parameter:
16 |     param_name type
17 | 
18 | type:
19 |     Any valid MySQL data type
20 | 
21 | characteristic:
22 |     COMMENT 'string'
23 |   | LANGUAGE SQL
24 |   | [NOT] DETERMINISTIC
25 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
26 |   | SQL SECURITY { DEFINER | INVOKER }
27 | 
28 | routine_body:
29 |     Valid SQL routine statement


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-server:
--------------------------------------------------------------------------------
 1 | CREATE SERVER server_name
 2 |     FOREIGN DATA WRAPPER wrapper_name
 3 |     OPTIONS (option [, option] ...)
 4 | 
 5 | option:
 6 |   { HOST character-literal
 7 |   | DATABASE character-literal
 8 |   | USER character-literal
 9 |   | PASSWORD character-literal
10 |   | SOCKET character-literal
11 |   | OWNER character-literal
12 |   | PORT numeric-literal }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-tablespace:
--------------------------------------------------------------------------------
 1 | CREATE TABLESPACE tablespace_name
 2 | 
 3 |   InnoDB and NDB:
 4 |     ADD DATAFILE 'file_name'
 5 | 
 6 |   InnoDB only:
 7 |     [FILE_BLOCK_SIZE = value]
 8 | 
 9 |   NDB only:
10 |     USE LOGFILE GROUP logfile_group
11 |     [EXTENT_SIZE [=] extent_size]
12 |     [INITIAL_SIZE [=] initial_size]
13 |     [AUTOEXTEND_SIZE [=] autoextend_size]
14 |     [MAX_SIZE [=] max_size]
15 |     [NODEGROUP [=] nodegroup_id]
16 |     [WAIT]
17 |     [COMMENT [=] 'string']
18 | 
19 |   InnoDB and NDB:
20 |     [ENGINE [=] engine_name]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-trigger:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     TRIGGER trigger_name
 4 |     trigger_time trigger_event
 5 |     ON tbl_name FOR EACH ROW
 6 |     [trigger_order]
 7 |     trigger_body
 8 | 
 9 | trigger_time: { BEFORE | AFTER }
10 | 
11 | trigger_event: { INSERT | UPDATE | DELETE }
12 | 
13 | trigger_order: { FOLLOWS | PRECEDES } other_trigger_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-user:
--------------------------------------------------------------------------------
 1 | CREATE USER [IF NOT EXISTS]
 2 |     user [auth_option] [, user [auth_option]] ...
 3 |     [REQUIRE {NONE | tls_option [[AND] tls_option] ...}]
 4 |     [WITH resource_option [resource_option] ...]
 5 |     [password_option | lock_option] ...
 6 | 
 7 | user:
 8 |     (see Section 6.2.3, “Specifying Account Names”)
 9 | 
10 | auth_option: {
11 |     IDENTIFIED BY 'auth_string'
12 |   | IDENTIFIED WITH auth_plugin
13 |   | IDENTIFIED WITH auth_plugin BY 'auth_string'
14 |   | IDENTIFIED WITH auth_plugin AS 'hash_string'
15 |   | IDENTIFIED BY PASSWORD 'hash_string'
16 | }
17 | 
18 | tls_option: {
19 |    SSL
20 |  | X509
21 |  | CIPHER 'cipher'
22 |  | ISSUER 'issuer'
23 |  | SUBJECT 'subject'
24 | }
25 | 
26 | resource_option: {
27 |     MAX_QUERIES_PER_HOUR count
28 |   | MAX_UPDATES_PER_HOUR count
29 |   | MAX_CONNECTIONS_PER_HOUR count
30 |   | MAX_USER_CONNECTIONS count
31 | }
32 | 
33 | password_option: {
34 |     PASSWORD EXPIRE
35 |   | PASSWORD EXPIRE DEFAULT
36 |   | PASSWORD EXPIRE NEVER
37 |   | PASSWORD EXPIRE INTERVAL N DAY
38 | }
39 | 
40 | lock_option: {
41 |     ACCOUNT LOCK
42 |   | ACCOUNT UNLOCK
43 | }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/create-view:
--------------------------------------------------------------------------------
1 | CREATE
2 |     [OR REPLACE]
3 |     [ALGORITHM = {UNDEFINED | MERGE | TEMPTABLE}]
4 |     [DEFINER = { user | CURRENT_USER }]
5 |     [SQL SECURITY { DEFINER | INVOKER }]
6 |     VIEW view_name [(column_list)]
7 |     AS select_statement
8 |     [WITH [CASCADED | LOCAL] CHECK OPTION]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-database:
--------------------------------------------------------------------------------
1 | DROP {DATABASE | SCHEMA} [IF EXISTS] db_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-event:
--------------------------------------------------------------------------------
1 | DROP EVENT [IF EXISTS] event_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-function-udf:
--------------------------------------------------------------------------------
1 | DROP FUNCTION function_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-index:
--------------------------------------------------------------------------------
1 | DROP INDEX index_name ON tbl_name
2 |     [algorithm_option | lock_option] ...
3 | 
4 | algorithm_option:
5 |     ALGORITHM [=] {DEFAULT|INPLACE|COPY}
6 | 
7 | lock_option:
8 |     LOCK [=] {DEFAULT|NONE|SHARED|EXCLUSIVE}


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-procedure:
--------------------------------------------------------------------------------
1 | DROP {PROCEDURE | FUNCTION} [IF EXISTS] sp_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-server:
--------------------------------------------------------------------------------
1 | DROP SERVER [ IF EXISTS ] server_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-table:
--------------------------------------------------------------------------------
1 | DROP [TEMPORARY] TABLE [IF EXISTS]
2 |     tbl_name [, tbl_name] ...
3 |     [RESTRICT | CASCADE]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-tablespace:
--------------------------------------------------------------------------------
1 | DROP TABLESPACE tablespace_name
2 |     [ENGINE [=] engine_name]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-trigger:
--------------------------------------------------------------------------------
1 | DROP TRIGGER [IF EXISTS] [schema_name.]trigger_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-user:
--------------------------------------------------------------------------------
1 | DROP USER [IF EXISTS] user [, user] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/drop-view:
--------------------------------------------------------------------------------
1 | DROP VIEW [IF EXISTS]
2 |     view_name [, view_name] ...
3 |     [RESTRICT | CASCADE]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/grant:
--------------------------------------------------------------------------------
 1 | GRANT
 2 |     priv_type [(column_list)]
 3 |       [, priv_type [(column_list)]] ...
 4 |     ON [object_type] priv_level
 5 |     TO user [auth_option] [, user [auth_option]] ...
 6 |     [REQUIRE {NONE | tls_option [[AND] tls_option] ...}]
 7 |     [WITH {GRANT OPTION | resource_option} ...]
 8 | 
 9 | GRANT PROXY ON user
10 |     TO user [, user] ...
11 |     [WITH GRANT OPTION]
12 | 
13 | object_type: {
14 |     TABLE
15 |   | FUNCTION
16 |   | PROCEDURE
17 | }
18 | 
19 | priv_level: {
20 |     *
21 |   | *.*
22 |   | db_name.*
23 |   | db_name.tbl_name
24 |   | tbl_name
25 |   | db_name.routine_name
26 | }
27 | 
28 | user:
29 |     (see Section 6.2.3, “Specifying Account Names”)
30 | 
31 | auth_option: {
32 |     IDENTIFIED BY 'auth_string'
33 |   | IDENTIFIED WITH auth_plugin
34 |   | IDENTIFIED WITH auth_plugin BY 'auth_string'
35 |   | IDENTIFIED WITH auth_plugin AS 'hash_string'
36 |   | IDENTIFIED BY PASSWORD 'hash_string'
37 | }
38 | 
39 | tls_option: {
40 |     SSL
41 |   | X509
42 |   | CIPHER 'cipher'
43 |   | ISSUER 'issuer'
44 |   | SUBJECT 'subject'
45 | }
46 | 
47 | resource_option: {
48 |   | MAX_QUERIES_PER_HOUR count
49 |   | MAX_UPDATES_PER_HOUR count
50 |   | MAX_CONNECTIONS_PER_HOUR count
51 |   | MAX_USER_CONNECTIONS count
52 | }


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/optimize-table:
--------------------------------------------------------------------------------
1 | OPTIMIZE [NO_WRITE_TO_BINLOG | LOCAL]
2 |     TABLE tbl_name [, tbl_name] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/rename-table:
--------------------------------------------------------------------------------
1 | RENAME TABLE
2 |     tbl_name TO new_tbl_name
3 |     [, tbl_name2 TO new_tbl_name2] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/rename-user:
--------------------------------------------------------------------------------
1 | RENAME USER old_user TO new_user
2 |     [, old_user TO new_user] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/repair-table:
--------------------------------------------------------------------------------
1 | REPAIR [NO_WRITE_TO_BINLOG | LOCAL]
2 |     TABLE tbl_name [, tbl_name] ...
3 |     [QUICK] [EXTENDED] [USE_FRM]


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/truncate-table:
--------------------------------------------------------------------------------
1 | TRUNCATE [TABLE] tbl_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/5.7/xa-statements:
--------------------------------------------------------------------------------
 1 | XA {START|BEGIN} xid [JOIN|RESUME]
 2 | 
 3 | XA END xid [SUSPEND [FOR MIGRATE]]
 4 | 
 5 | XA PREPARE xid
 6 | 
 7 | XA COMMIT xid [ONE PHASE]
 8 | 
 9 | XA ROLLBACK xid
10 | 
11 | XA RECOVER [CONVERT XID]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-database:
--------------------------------------------------------------------------------
1 | ALTER {DATABASE | SCHEMA} [db_name]
2 |     alter_specification ...
3 | 
4 | alter_specification:
5 |     [DEFAULT] CHARACTER SET [=] charset_name
6 |   | [DEFAULT] COLLATE [=] collation_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-event:
--------------------------------------------------------------------------------
1 | ALTER
2 |     [DEFINER = { user | CURRENT_USER }]
3 |     EVENT event_name
4 |     [ON SCHEDULE schedule]
5 |     [ON COMPLETION [NOT] PRESERVE]
6 |     [RENAME TO new_event_name]
7 |     [ENABLE | DISABLE | DISABLE ON SLAVE]
8 |     [COMMENT 'string']
9 |     [DO event_body]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-function:
--------------------------------------------------------------------------------
1 | ALTER FUNCTION func_name [characteristic ...]
2 | 
3 | characteristic:
4 |     COMMENT 'string'
5 |   | LANGUAGE SQL
6 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
7 |   | SQL SECURITY { DEFINER | INVOKER }


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-instance:
--------------------------------------------------------------------------------
1 | ALTER INSTANCE ROTATE INNODB MASTER KEY


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-procedure:
--------------------------------------------------------------------------------
1 | ALTER PROCEDURE proc_name [characteristic ...]
2 | 
3 | characteristic:
4 |     COMMENT 'string'
5 |   | LANGUAGE SQL
6 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
7 |   | SQL SECURITY { DEFINER | INVOKER }


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-resource-group:
--------------------------------------------------------------------------------
1 | ALTER RESOURCE GROUP group_name
2 |     [VCPU [=] vcpu_spec [, vcpu_spec] ...]
3 |     [THREAD_PRIORITY [=] N]
4 |     [ENABLE|DISABLE [FORCE]]
5 | 
6 | vcpu_spec: {N | M - N}


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-server:
--------------------------------------------------------------------------------
1 | ALTER SERVER  server_name
2 |     OPTIONS (option [, option] ...)


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-tablespace:
--------------------------------------------------------------------------------
1 | ALTER TABLESPACE tablespace_name
2 |     [RENAME TO tablespace_name] 
3 |     [ENCRYPTION [=] {'Y' | 'N'}]
4 |     [ENGINE [=] engine_name]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-user:
--------------------------------------------------------------------------------
 1 | ALTER USER [IF EXISTS]
 2 |     user [auth_option] [, user [auth_option]] ...
 3 |     [REQUIRE {NONE | tls_option [[AND] tls_option] ...}]
 4 |     [WITH resource_option [resource_option] ...]
 5 |     [password_option | lock_option] ...
 6 | 
 7 | ALTER USER [IF EXISTS]
 8 |     USER() IDENTIFIED BY 'auth_string' [REPLACE 'current_auth_string']
 9 | 
10 | ALTER USER [IF EXISTS]
11 |     user DEFAULT ROLE
12 |     {NONE | ALL | role [, role ] ...}
13 | 
14 | user:
15 |     (see Section 6.2.4, “Specifying Account Names”)
16 | 
17 | auth_option: {
18 |     IDENTIFIED BY 'auth_string' [REPLACE 'current_auth_string']
19 |   | IDENTIFIED WITH auth_plugin
20 |   | IDENTIFIED WITH auth_plugin BY 'auth_string' [REPLACE 'current_auth_string']
21 |   | IDENTIFIED WITH auth_plugin AS 'hash_string'
22 | }
23 | 
24 | tls_option: {
25 |    SSL
26 |  | X509
27 |  | CIPHER 'cipher'
28 |  | ISSUER 'issuer'
29 |  | SUBJECT 'subject'
30 | }
31 | 
32 | resource_option: {
33 |     MAX_QUERIES_PER_HOUR count
34 |   | MAX_UPDATES_PER_HOUR count
35 |   | MAX_CONNECTIONS_PER_HOUR count
36 |   | MAX_USER_CONNECTIONS count
37 | }
38 | 
39 | password_option: {
40 |     PASSWORD EXPIRE [DEFAULT | NEVER | INTERVAL N DAY]
41 |   | PASSWORD HISTORY {DEFAULT | N}
42 |   | PASSWORD REUSE INTERVAL {DEFAULT | N DAY}
43 |   | PASSWORD REQUIRE CURRENT [DEFAULT | OPTIONAL]
44 | }
45 | 
46 | lock_option: {
47 |     ACCOUNT LOCK
48 |   | ACCOUNT UNLOCK
49 | }


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/alter-view:
--------------------------------------------------------------------------------
1 | ALTER
2 |     [ALGORITHM = {UNDEFINED | MERGE | TEMPTABLE}]
3 |     [DEFINER = { user | CURRENT_USER }]
4 |     [SQL SECURITY { DEFINER | INVOKER }]
5 |     VIEW view_name [(column_list)]
6 |     AS select_statement
7 |     [WITH [CASCADED | LOCAL] CHECK OPTION]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/analyze-table:
--------------------------------------------------------------------------------
 1 | ANALYZE [NO_WRITE_TO_BINLOG | LOCAL]
 2 |     TABLE tbl_name [, tbl_name] ...
 3 | 
 4 | ANALYZE [NO_WRITE_TO_BINLOG | LOCAL]
 5 |     TABLE tbl_name
 6 |     UPDATE HISTOGRAM ON col_name [, col_name] ...
 7 |         [WITH N BUCKETS]
 8 | 
 9 | ANALYZE [NO_WRITE_TO_BINLOG | LOCAL]
10 |     TABLE tbl_name
11 |     DROP HISTOGRAM ON col_name [, col_name] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/cache-index:
--------------------------------------------------------------------------------
 1 | CACHE INDEX
 2 |   tbl_index_list [, tbl_index_list] ...
 3 |   [PARTITION (partition_list | ALL)]
 4 |   IN key_cache_name
 5 | 
 6 | tbl_index_list:
 7 |   tbl_name [[INDEX|KEY] (index_name[, index_name] ...)]
 8 | 
 9 | partition_list:
10 |   partition_name[, partition_name][, ...]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/checksum-table:
--------------------------------------------------------------------------------
1 | CHECKSUM TABLE tbl_name [, tbl_name] ... [QUICK | EXTENDED]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-database:
--------------------------------------------------------------------------------
1 | CREATE {DATABASE | SCHEMA} [IF NOT EXISTS] db_name
2 |     [create_specification] ...
3 | 
4 | create_specification:
5 |     [DEFAULT] CHARACTER SET [=] charset_name
6 |   | [DEFAULT] COLLATE [=] collation_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-event:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     EVENT
 4 |     [IF NOT EXISTS]
 5 |     event_name
 6 |     ON SCHEDULE schedule
 7 |     [ON COMPLETION [NOT] PRESERVE]
 8 |     [ENABLE | DISABLE | DISABLE ON SLAVE]
 9 |     [COMMENT 'string']
10 |     DO event_body;
11 | 
12 | schedule:
13 |     AT timestamp [+ INTERVAL interval] ...
14 |   | EVERY interval
15 |     [STARTS timestamp [+ INTERVAL interval] ...]
16 |     [ENDS timestamp [+ INTERVAL interval] ...]
17 | 
18 | interval:
19 |     quantity {YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE |
20 |               WEEK | SECOND | YEAR_MONTH | DAY_HOUR | DAY_MINUTE |
21 |               DAY_SECOND | HOUR_MINUTE | HOUR_SECOND | MINUTE_SECOND}


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-function-udf:
--------------------------------------------------------------------------------
1 | CREATE [AGGREGATE] FUNCTION function_name
2 |     RETURNS {STRING|INTEGER|REAL|DECIMAL}
3 |     SONAME shared_library_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-index:
--------------------------------------------------------------------------------
 1 | CREATE [UNIQUE | FULLTEXT | SPATIAL] INDEX index_name
 2 |     [index_type]
 3 |     ON tbl_name (key_part,...)
 4 |     [index_option]
 5 |     [algorithm_option | lock_option] ...
 6 | 
 7 | key_part: {col_name [(length)] | (expr)} [ASC | DESC]
 8 | 
 9 | index_option:
10 |     KEY_BLOCK_SIZE [=] value
11 |   | index_type
12 |   | WITH PARSER parser_name
13 |   | COMMENT 'string'
14 |   | {VISIBLE | INVISIBLE}
15 | 
16 | index_type:
17 |     USING {BTREE | HASH}
18 | 
19 | algorithm_option:
20 |     ALGORITHM [=] {DEFAULT | INPLACE | COPY}
21 | 
22 | lock_option:
23 |     LOCK [=] {DEFAULT | NONE | SHARED | EXCLUSIVE}


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-procedure:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     PROCEDURE sp_name ([proc_parameter[,...]])
 4 |     [characteristic ...] routine_body
 5 | 
 6 | CREATE
 7 |     [DEFINER = { user | CURRENT_USER }]
 8 |     FUNCTION sp_name ([func_parameter[,...]])
 9 |     RETURNS type
10 |     [characteristic ...] routine_body
11 | 
12 | proc_parameter:
13 |     [ IN | OUT | INOUT ] param_name type
14 | 
15 | func_parameter:
16 |     param_name type
17 | 
18 | type:
19 |     Any valid MySQL data type
20 | 
21 | characteristic:
22 |     COMMENT 'string'
23 |   | LANGUAGE SQL
24 |   | [NOT] DETERMINISTIC
25 |   | { CONTAINS SQL | NO SQL | READS SQL DATA | MODIFIES SQL DATA }
26 |   | SQL SECURITY { DEFINER | INVOKER }
27 | 
28 | routine_body:
29 |     Valid SQL routine statement


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-resource-group:
--------------------------------------------------------------------------------
1 | CREATE RESOURCE GROUP group_name
2 |     TYPE = {SYSTEM|USER}
3 |     [VCPU [=] vcpu_spec [, vcpu_spec] ...]
4 |     [THREAD_PRIORITY [=] N]
5 |     [ENABLE|DISABLE]
6 | 
7 | vcpu_spec: {N | M - N}


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-role:
--------------------------------------------------------------------------------
1 | CREATE ROLE [IF NOT EXISTS] role [, role ] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-server:
--------------------------------------------------------------------------------
 1 | CREATE SERVER server_name
 2 |     FOREIGN DATA WRAPPER wrapper_name
 3 |     OPTIONS (option [, option] ...)
 4 | 
 5 | option:
 6 |   { HOST character-literal
 7 |   | DATABASE character-literal
 8 |   | USER character-literal
 9 |   | PASSWORD character-literal
10 |   | SOCKET character-literal
11 |   | OWNER character-literal
12 |   | PORT numeric-literal }


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-spatial-reference-system:
--------------------------------------------------------------------------------
 1 | CREATE OR REPLACE SPATIAL REFERENCE SYSTEM
 2 |     srid srs_attribute ...
 3 | 
 4 | CREATE SPATIAL REFERENCE SYSTEM
 5 |     [IF NOT EXISTS]
 6 |     srid srs_attribute ...
 7 | 
 8 | srs_attribute: {
 9 |     NAME 'srs_name'
10 |   | DEFINITION 'definition'
11 |   | ORGANIZATION 'org_name' IDENTIFIED BY org_id
12 |   | DESCRIPTION 'description'
13 | }
14 | 
15 | srid, org_id: 32-bit unsigned integer


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-tablespace:
--------------------------------------------------------------------------------
1 | CREATE TABLESPACE tablespace_name
2 |     ADD DATAFILE 'file_name'
3 |     [FILE_BLOCK_SIZE = value]
4 |     [ENCRYPTION [=] {'Y' | 'N'}]
5 |     [ENGINE [=] engine_name]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-trigger:
--------------------------------------------------------------------------------
 1 | CREATE
 2 |     [DEFINER = { user | CURRENT_USER }]
 3 |     TRIGGER trigger_name
 4 |     trigger_time trigger_event
 5 |     ON tbl_name FOR EACH ROW
 6 |     [trigger_order]
 7 |     trigger_body
 8 | 
 9 | trigger_time: { BEFORE | AFTER }
10 | 
11 | trigger_event: { INSERT | UPDATE | DELETE }
12 | 
13 | trigger_order: { FOLLOWS | PRECEDES } other_trigger_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-user:
--------------------------------------------------------------------------------
 1 | CREATE USER [IF NOT EXISTS]
 2 |     user [auth_option] [, user [auth_option]] ...
 3 |     DEFAULT ROLE role [, role ] ...
 4 |     [REQUIRE {NONE | tls_option [[AND] tls_option] ...}]
 5 |     [WITH resource_option [resource_option] ...]
 6 |     [password_option | lock_option] ...
 7 | 
 8 | user:
 9 |     (see Section 6.2.4, “Specifying Account Names”)
10 | 
11 | auth_option: {
12 |     IDENTIFIED BY 'auth_string'
13 |   | IDENTIFIED WITH auth_plugin
14 |   | IDENTIFIED WITH auth_plugin BY 'auth_string'
15 |   | IDENTIFIED WITH auth_plugin AS 'hash_string'
16 | }
17 | 
18 | tls_option: {
19 |    SSL
20 |  | X509
21 |  | CIPHER 'cipher'
22 |  | ISSUER 'issuer'
23 |  | SUBJECT 'subject'
24 | }
25 | 
26 | resource_option: {
27 |     MAX_QUERIES_PER_HOUR count
28 |   | MAX_UPDATES_PER_HOUR count
29 |   | MAX_CONNECTIONS_PER_HOUR count
30 |   | MAX_USER_CONNECTIONS count
31 | }
32 | 
33 | password_option: {
34 |     PASSWORD EXPIRE [DEFAULT | NEVER | INTERVAL N DAY]
35 |   | PASSWORD HISTORY {DEFAULT | N}
36 |   | PASSWORD REUSE INTERVAL {DEFAULT | N DAY}
37 |   | PASSWORD REQUIRE CURRENT [DEFAULT | OPTIONAL]
38 | }
39 | 
40 | lock_option: {
41 |     ACCOUNT LOCK
42 |   | ACCOUNT UNLOCK
43 | }


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/create-view:
--------------------------------------------------------------------------------
1 | CREATE
2 |     [OR REPLACE]
3 |     [ALGORITHM = {UNDEFINED | MERGE | TEMPTABLE}]
4 |     [DEFINER = { user | CURRENT_USER }]
5 |     [SQL SECURITY { DEFINER | INVOKER }]
6 |     VIEW view_name [(column_list)]
7 |     AS select_statement
8 |     [WITH [CASCADED | LOCAL] CHECK OPTION]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-database:
--------------------------------------------------------------------------------
1 | DROP {DATABASE | SCHEMA} [IF EXISTS] db_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-event:
--------------------------------------------------------------------------------
1 | DROP EVENT [IF EXISTS] event_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-function-udf:
--------------------------------------------------------------------------------
1 | DROP FUNCTION function_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-index:
--------------------------------------------------------------------------------
1 | DROP INDEX index_name ON tbl_name
2 |     [algorithm_option | lock_option] ...
3 | 
4 | algorithm_option:
5 |     ALGORITHM [=] {DEFAULT|INPLACE|COPY}
6 | 
7 | lock_option:
8 |     LOCK [=] {DEFAULT|NONE|SHARED|EXCLUSIVE}


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-procedure:
--------------------------------------------------------------------------------
1 | DROP {PROCEDURE | FUNCTION} [IF EXISTS] sp_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-resource-group:
--------------------------------------------------------------------------------
1 | DROP RESOURCE GROUP group_name [FORCE]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-role:
--------------------------------------------------------------------------------
1 | DROP ROLE [IF EXISTS] role [, role ] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-server:
--------------------------------------------------------------------------------
1 | DROP SERVER [ IF EXISTS ] server_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-spatial-reference-system:
--------------------------------------------------------------------------------
1 | DROP SPATIAL REFERENCE SYSTEM
2 |     [IF EXISTS]
3 |     srid
4 | 
5 | srid: 32-bit unsigned integer


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-table:
--------------------------------------------------------------------------------
1 | DROP [TEMPORARY] TABLE [IF EXISTS]
2 |     tbl_name [, tbl_name] ...
3 |     [RESTRICT | CASCADE]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-tablespace:
--------------------------------------------------------------------------------
1 | DROP TABLESPACE tablespace_name
2 |     [ENGINE [=] engine_name]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-trigger:
--------------------------------------------------------------------------------
1 | DROP TRIGGER [IF EXISTS] [schema_name.]trigger_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-user:
--------------------------------------------------------------------------------
1 | DROP USER [IF EXISTS] user [, user] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/drop-view:
--------------------------------------------------------------------------------
1 | DROP VIEW [IF EXISTS]
2 |     view_name [, view_name] ...
3 |     [RESTRICT | CASCADE]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/grant:
--------------------------------------------------------------------------------
 1 | GRANT
 2 |     priv_type [(column_list)]
 3 |       [, priv_type [(column_list)]] ...
 4 |     ON [object_type] priv_level
 5 |     TO user_or_role [, user_or_role] ...
 6 |     [WITH GRANT OPTION]
 7 | 
 8 | GRANT PROXY ON user_or_role
 9 |     TO user_or_role [, user_or_role] ...
10 |     [WITH GRANT OPTION]
11 | 
12 | GRANT role [, role] ...
13 |     TO user_or_role [, user_or_role] ...
14 |     [WITH ADMIN OPTION]
15 | 
16 | object_type: {
17 |     TABLE
18 |   | FUNCTION
19 |   | PROCEDURE
20 | }
21 | 
22 | priv_level: {
23 |     *
24 |   | *.*
25 |   | db_name.*
26 |   | db_name.tbl_name
27 |   | tbl_name
28 |   | db_name.routine_name
29 | }
30 | 
31 | user_or_role: {
32 |     user
33 |   | role
34 | }
35 | 
36 | user:
37 |     (see Section 6.2.4, “Specifying Account Names”)
38 | 
39 | role:
40 |     (see Section 6.2.5, “Specifying Role Names”)


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/optimize-table:
--------------------------------------------------------------------------------
1 | OPTIMIZE [NO_WRITE_TO_BINLOG | LOCAL]
2 |     TABLE tbl_name [, tbl_name] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/rename-table:
--------------------------------------------------------------------------------
1 | RENAME TABLE
2 |     tbl_name TO new_tbl_name
3 |     [, tbl_name2 TO new_tbl_name2] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/rename-user:
--------------------------------------------------------------------------------
1 | RENAME USER old_user TO new_user
2 |     [, old_user TO new_user] ...


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/repair-table:
--------------------------------------------------------------------------------
1 | REPAIR [NO_WRITE_TO_BINLOG | LOCAL]
2 |     TABLE tbl_name [, tbl_name] ...
3 |     [QUICK] [EXTENDED] [USE_FRM]


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/truncate-table:
--------------------------------------------------------------------------------
1 | TRUNCATE [TABLE] tbl_name


--------------------------------------------------------------------------------
/src/test/resources/ddl/8.0/xa-statements:
--------------------------------------------------------------------------------
 1 | XA {START|BEGIN} xid [JOIN|RESUME]
 2 | 
 3 | XA END xid [SUSPEND [FOR MIGRATE]]
 4 | 
 5 | XA PREPARE xid
 6 | 
 7 | XA COMMIT xid [ONE PHASE]
 8 | 
 9 | XA ROLLBACK xid
10 | 
11 | XA RECOVER [CONVERT XID]


--------------------------------------------------------------------------------
/src/test/resources/ddl/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | gem 'oga'
4 | 
5 | 


--------------------------------------------------------------------------------
/src/test/resources/ddl/Gemfile.lock:
--------------------------------------------------------------------------------
 1 | GEM
 2 |   remote: https://rubygems.org/
 3 |   specs:
 4 |     ansi (1.5.0)
 5 |     ast (2.4.0)
 6 |     oga (2.15)
 7 |       ast
 8 |       ruby-ll (~> 2.1)
 9 |     ruby-ll (2.1.2)
10 |       ansi
11 |       ast
12 | 
13 | PLATFORMS
14 |   ruby
15 | 
16 | DEPENDENCIES
17 |   oga
18 | 
19 | BUNDLED WITH
20 |    1.13.5
21 | 


--------------------------------------------------------------------------------
/src/test/resources/ddl/get_ddl.rb:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env ruby
 2 | 
 3 | require 'bundler/setup'
 4 | require 'oga'
 5 | require 'net/http'
 6 | 
 7 | VERSION=ARGV[0]
 8 | URL=ARGV[1]
 9 | 
10 | def parse_url(url)
11 |   html = Net::HTTP.get(URI(url))
12 |   document = Oga.parse_html(html)
13 | end
14 | 
15 | def extract_code(link_name, doc)
16 |   code = doc.at_css("code")
17 |   return unless code
18 |   filename = VERSION + "/" + link_name.sub(/\..*$?/, '')
19 |   File.open(filename, "w+") do |file|
20 |     file.write(code.text)
21 |   end
22 | end
23 | 
24 | 
25 | base_doc = parse_url(URL)
26 | base_doc.css(".docs-submenu .docs-sidebar-nav-link a").each do |link|
27 |   new_link = URL.sub(%r{/[^/]*$}, '') + "/" + link.attr("href").to_s
28 | 
29 |   sub_doc = parse_url(new_link)
30 |   extract_code(link.attr("href").to_s, sub_doc)
31 | end
32 | 
33 | 


--------------------------------------------------------------------------------
/src/test/resources/json/all-encrypted-user-insert.json:
--------------------------------------------------------------------------------
1 | {"encrypted":{"bytes":"iZssjWfzS0NlqIj82ddpvoQeKSx4D3GIPSCgjdkpgQlCWzN2p3VVZOn3Oj1x4w+a6dVhoFmllWxBK6aAkdVK9t6Vt1+um6lWwSeXNQIL/RbknW5Q8I9emm5bC1Dd1LftBuX/1Uw0wjbsq8Qt3HErvmmiIMe2S27EIWshvBnmw9MibryjLD0brvIbFFxwDuSQuVA4OFyV9TN32N/ZXiBwIA==","iv":"XXs6AePsXJWAAIrKyLlR0g=="}}


--------------------------------------------------------------------------------
/src/test/resources/json/all-encrypted-user-update.json:
--------------------------------------------------------------------------------
1 | {"encrypted": {"bytes":"pvqj6Pf1dnp+FbIgiOpIMWuKXNz3MV1B2+hBttWMAPQFYxafnrYfVW43/aVn5myviAsXV1cFQH0vAbrk/Lmn8WOMPgN8789KO4coTWuZbPAJMBgHHv2NVsucf0RyBFseXkI09DgKo0fm7S8z7LJxbiwXKqrur9CeNZBpGoqJuxbkVya+qeeSFJdXczn08t3g9xdXvu3R0hMw3cdtb03FeO3yP3ciZeJ+IkY9XPKGjNQqSfp439hybXGDvpNGSh2h","iv":"LQz0qFlOO5VpfQ8EMlBTUg=="}}


--------------------------------------------------------------------------------
/src/test/resources/json/encrypted-user-insert.json:
--------------------------------------------------------------------------------
1 | {"database":"shard_1","table":"minimal","type":"insert","ts":1504585129,"xid":161,"commit":true,"encrypted":{"iv":"lqiXoTdz6jed3XgJPpa7EQ==","bytes":"1soc4leskiIm6yuT2D49VA3AYVKCvN+0wh+8d1iwSZETK7N2pG4HDbqnVpJUUCOaKjpcPlP7Sc7Z3SPhGD5JeA=="}}


--------------------------------------------------------------------------------
/src/test/resources/json/encrypted-user-update.json:
--------------------------------------------------------------------------------
1 | {"database":"shard_1","table":"minimal","type":"update","ts":1504584795,"xid":162,"commit":true,"encrypted":{"iv":"UIWqg0JEwEGHUXyXHgPv0Q==","bytes":"g21dANWEADe8gat28uMten7Yx0NSFzy/SaTdUxXXPUL+fXwvhL5v/UdgJugkHPfc2e2GniQSPgbdB6Mpn+IFFnw8/bLa8ndIF/mizXO9U7SSnKKdr/X+GE4AylpvUa4+"}}


--------------------------------------------------------------------------------
/src/test/resources/json/user-insert.json:
--------------------------------------------------------------------------------
1 | {"database":"MyDatabase","table":"User","type":"insert","ts":1486439516,"xid":5694,"commit":true,"data":{"UserID":20,"FirstName":"Fiz","MiddleName":null,"LastName":"Buz","DoubleField":4500000001.512456777,"Version":1486703131000}}
2 | 


--------------------------------------------------------------------------------
/src/test/resources/json/user-update.json:
--------------------------------------------------------------------------------
1 | {"database":"MyDatabase","table":"User","type":"update","ts":1486439516,"xid":5694,"commit":true,"data":{"UserID":20,"FirstName":"Fiz","MiddleName":null,"LastName":"Buz","Version":1486703131000},"old":{"backup_date":"2017-02-08 02:00:05","FirstName":"Foo","DoubleField":4500000001.512456777,"LastName":"Bar"}}
2 | 


--------------------------------------------------------------------------------
/src/test/resources/log4j2.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <Configuration status="WARN">
 3 |   <Appenders>
 4 |     <File name="TestLog" fileName="test.log">
 5 |       <PatternLayout pattern="[%T] %d{HH:mm:ss,SSS} %-5level %logger{1} - %msg%n"/>
 6 |     </File>
 7 |   </Appenders>
 8 |   <Loggers>
 9 |     <Root level="debug">
10 |       <AppenderRef ref="TestLog"/>
11 |     </Root>
12 |    <Logger name="com.mchange.v2" level="error" additivity="false">
13 |       <AppenderRef ref="TestLog"/>
14 |     </Logger>
15 |   </Loggers>
16 | </Configuration>
17 | 


--------------------------------------------------------------------------------
/src/test/resources/logging.properties:
--------------------------------------------------------------------------------
1 | handlers = org.slf4j.bridge.SLF4JBridgeHandler
2 | 
3 | 


--------------------------------------------------------------------------------
/src/test/resources/scripting/test-no-state-suppress-row.js:
--------------------------------------------------------------------------------
1 | function process_row(r) {
2 |   r.suppress();
3 | }
4 | 


--------------------------------------------------------------------------------
/src/test/resources/scripting/test-set-state.js:
--------------------------------------------------------------------------------
1 | function process_row(r, state) {
2 |   state.put("mykey", "myvalue");
3 | }
4 | 
5 | function process_ddl(ddl, state) {
6 |   var num = parseInt(state.get("number"));
7 |   state.put("number", Number(num + 1).toFixed(0));
8 | }


--------------------------------------------------------------------------------
/src/test/resources/sql/ddl/mysql-test-errors.sql:
--------------------------------------------------------------------------------
 1 | ALTER TABLE ti1 CHECKSUM 1
 2 | ALTER TABLE tm1 CHECKSUM 1
 3 | CREATE TABLE "table_25930_b" (   """blah"" - 1" bigint(12) DEFAULT NULL )
 4 | CREATE TABLE t1 ( c01 BIT, c02 BIT(64), c03 TINYINT, c04 TINYINT UNSIGNED, c05 TINYINT ZEROFILL, c06 BOOL, c07 SMALLINT, c08 SMALLINT UNSIGNED, c09 SMALLINT ZEROFILL, c10 MEDIUMINT, c11 MEDIUMINT UNSIGNED, c12 MEDIUMINT ZEROFILL, c13 INT, c14 INT UNSIGNED, c15 INT ZEROFILL, c16 BIGINT, c17 BIGINT UNSIGNED, c18 BIGINT ZEROFILL, c19 FLOAT, c20 FLOAT UNSIGNED, c21 FLOAT ZEROFILL, c22 DOUBLE, c23 DOUBLE UNSIGNED, c24 DOUBLE ZEROFILL, c25 DECIMAL, c26 DECIMAL UNSIGNED, c27 DECIMAL ZEROFILL,
 5 | CREATE TABLE t1 ( c1 INT, c2 VARCHAR(300), KEY (c1) KEY_BLOCK_SIZE 1024, KEY (c2) KEY_BLOCK_SIZE 8192 )
 6 | alter table t1 add f2 enum(0xFFFF)
 7 | create table t1 ( min_num   dec(6,6)     default .000001)
 8 | create table t1 ( min_num   dec(6,6)     default 0.000001)
 9 | create table t1 ("t1 column" int)
10 | create table t1 (a enum(0xE4, '1', '2') not null default 0xE4)
11 | create table t1 (c nchar varying(10))
12 | create table t1 (t1.index int)
13 | create table t1(a char character set cp1251 default _koi8r 0xFF)
14 | create table t1(c enum(0x9353,0x9373) character set sjis)
15 | create table t1(t1.name int)
16 | create table t2 as select * from t1
17 | create table t2(test.t2.name int)
18 | create table test.no_index_tab ( a varchar(255) not null, b int not null) engine = merge union = (test.no_index_tab_1,test.no_index_tab_2) insert_method = first
19 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-js-filters:
--------------------------------------------------------------------------------
1 | truncate maxwell.bootstrap;
2 | create table shard_1.bootstrap_js (id int auto_increment primary key, a varchar(255))
3 | insert into shard_1.bootstrap_js set a = 'abcdef'
4 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'bootstrap_js'
5 | 
6 | -> { database: "shard_1", table: "bootstrap_js", type:"bootstrap-insert", data: { id: 1, a: "ABCDEF" } }
7 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-json-type:
--------------------------------------------------------------------------------
1 | truncate maxwell.bootstrap;
2 | use shard_1;
3 | create table if not exists bootstrap_json_type ( j json );
4 | insert into shard_1.bootstrap_json_type set j = '{"hello": "world"}';
5 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'bootstrap_json_type'
6 | 
7 | -> { database: "shard_1", table: "bootstrap_json_type", type: "bootstrap-insert", data: { j: {hello: "world"} } }
8 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-multiple-row:
--------------------------------------------------------------------------------
 1 | insert into minimal set account_id = 1, text_field='hello';
 2 | insert into minimal set account_id = 2, text_field='bonjour';
 3 | insert into minimal set account_id = 3, text_field='goeiedag';
 4 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'minimal';
 5 | -> { database: "shard_1", table: "minimal", type: "insert",          data: {id: 1, account_id: 1, text_field: "hello"} }
 6 | -> { database: "shard_1", table: "minimal", type: "insert",          data: {id: 2, account_id: 2, text_field: "bonjour"} }
 7 | -> { database: "shard_1", table: "minimal", type: "insert",          data: {id: 3, account_id: 3, text_field: "goeiedag"} }
 8 | -> { database: "maxwell", table: "bootstrap", type: "insert",        data: {id: 1, database_name: "shard_1", table_name: "minimal", is_complete: 0, inserted_rows: 0, binlog_position:0, total_rows: 0, created_at: null, started_at: null, completed_at: null, binlog_file: null, where_clause: null, client_id: "maxwell", comment: null }}
 9 | -> { database: "shard_1", table: "minimal", type: "bootstrap-start", data: {} }
10 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert",          data: {id: 1, account_id: 1, text_field: "hello" } }
11 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert",          data: {id: 2, account_id: 2, text_field: "bonjour"} }
12 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert",          data: {id: 3, account_id: 3, text_field: "goeiedag"} }
13 | -> { database: "shard_1", table: "minimal", type: "bootstrap-complete", data: {} }
14 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-multiple-row-with-whereclause:
--------------------------------------------------------------------------------
 1 | truncate table maxwell.bootstrap;
 2 | truncate table shard_1.minimal;
 3 | 
 4 | insert into minimal set account_id = 1, text_field='hello';
 5 | insert into minimal set account_id = 2, text_field='bonjour';
 6 | insert into minimal set account_id = 3, text_field='goeiedag';
 7 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'minimal', where_clause = 'id > 1';
 8 | -> { database: "maxwell", table: "bootstrap", type: "insert",        data: {id: 1, database_name: "shard_1", table_name: "minimal", is_complete: 0, inserted_rows: 0, binlog_position:0, total_rows: 0, created_at: null, started_at: null, completed_at: null, binlog_file: null, where_clause: "id > 1", client_id: "maxwell", comment: null }}
 9 | -> { database: "shard_1", table: "minimal", type: "bootstrap-start", data: {} }
10 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert",          data: {id: 2, account_id: 2, text_field: "bonjour"} }
11 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert",          data: {id: 3, account_id: 3, text_field: "goeiedag"} }
12 | -> { database: "shard_1", table: "minimal", type: "bootstrap-complete", data: {} }
13 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-no-pk:
--------------------------------------------------------------------------------
 1 | truncate table maxwell.bootstrap;
 2 | DROP TABLE IF EXISTS shard_1.table_no_pk;
 3 | CREATE TABLE IF NOT EXISTS shard_1.table_no_pk(account_id int unsigned, text_field varchar(255));
 4 | insert into table_no_pk set account_id = 1, text_field='hello';
 5 | insert into table_no_pk set account_id = 2, text_field='bonjour';
 6 | insert into table_no_pk set account_id = 3, text_field='goeiedag';
 7 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'table_no_pk';
 8 | 
 9 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 1, text_field: "hello"} }
10 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 2, text_field: "bonjour"} }
11 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 3, text_field: "goeiedag"} }
12 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-start",  data: {} }
13 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 1, text_field: "hello"} }
14 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 2, text_field: "bonjour"} }
15 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-insert",  data: {account_id: 3, text_field: "goeiedag"} }
16 | -> { database: "shard_1", table: "table_no_pk", type: "bootstrap-complete",  data: {} }
17 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-null-values:
--------------------------------------------------------------------------------
1 | truncate maxwell.bootstrap;
2 | create table shard_1.null_bootstraps (id int auto_increment primary key, txt varchar(255) null default null)
3 | insert into shard_1.null_bootstraps set txt = NULL
4 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'null_bootstraps'
5 | 
6 | -> { database: "shard_1", table: "null_bootstraps", type: "bootstrap-start", data: {} }
7 | -> { database: "shard_1", table: "null_bootstraps", type:"bootstrap-insert", data: { id: 1, txt: null } }
8 | -> { database: "shard_1", table: "null_bootstraps", type: "bootstrap-complete", data: {}}
9 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-second-server:
--------------------------------------------------------------------------------
1 | insert into maxwell.bootstrap set database_name = 'test_other', table_name = 'other';
2 | 
3 | -> { database: "test_other", table: "other", type: "bootstrap-insert",  data: {id: 1} }
4 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-single-row:
--------------------------------------------------------------------------------
 1 | truncate table maxwell.bootstrap;
 2 | insert into minimal set account_id = 1, text_field='hello'
 3 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'minimal'
 4 | 
 5 | -> { database: "shard_1", table: "minimal", type:"insert", data: { id: 1, account_id: 1, text_field: "hello"} }
 6 | -> { database: "maxwell", table: "bootstrap",type: "insert", data: { database_name: "shard_1", inserted_rows: 0, total_rows: 0, binlog_position: 0, id: 1, is_complete: 0, table_name: "minimal", created_at: null, started_at: null, completed_at: null, binlog_file: null, where_clause: null, client_id: "maxwell", comment: null  } }
 7 | -> { database: "shard_1", table: "minimal", type: "bootstrap-start", data: {} }
 8 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert", data: { id: 1, account_id: 1, text_field: "hello"} }
 9 | -> { database: "shard_1", table: "minimal", type: "bootstrap-complete", data: {}}
10 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-whitelist:
--------------------------------------------------------------------------------
 1 | truncate maxwell.bootstrap;
 2 | insert into shard_1.minimal set account_id = 1, text_field='hello'
 3 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'minimal'
 4 | 
 5 | -> { database: "shard_1", table: "minimal", type:"insert", data: { id: 1, account_id: 1, text_field: "hello"} }
 6 | -> { database: "maxwell", table: "bootstrap",type: "insert", data: { database_name: "shard_1", inserted_rows: 0, total_rows: 0, binlog_position: 0, id: 1, is_complete: 0, table_name: "minimal", created_at: null, started_at: null, completed_at: null, binlog_file: null, where_clause: null, client_id: "maxwell", comment: null  } }
 7 | -> { database: "shard_1", table: "minimal", type: "bootstrap-start", data: {} }
 8 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert", data: { id: 1, account_id: 1, text_field: "hello"} }
 9 | -> { database: "shard_1", table: "minimal", type: "bootstrap-complete", data: {}}
10 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/bootstrap-with-comment:
--------------------------------------------------------------------------------
1 | truncate table maxwell.bootstrap;
2 | insert into minimal set account_id = 1, text_field='hello'
3 | insert into maxwell.bootstrap set database_name = 'shard_1', table_name = 'minimal', comment = 'some text'
4 | 
5 | -> { database: "shard_1", table: "minimal", type:"insert", data: { id: 1, account_id: 1, text_field: "hello"} }
6 | -> { database: "maxwell", table: "bootstrap",type: "insert", data: { database_name: "shard_1", inserted_rows: 0, total_rows: 0, binlog_position: 0, id: 1, is_complete: 0, table_name: "minimal", created_at: null, started_at: null, completed_at: null, binlog_file: null, where_clause: null, client_id: "maxwell", comment: "some text"  } }
7 | -> { database: "shard_1", table: "minimal", type: "bootstrap-start", comment: "some text", data: {} }
8 | -> { database: "shard_1", table: "minimal", type: "bootstrap-insert", comment: "some text", data: { id: 1, account_id: 1, text_field: "hello"} }
9 | -> { database: "shard_1", table: "minimal", type: "bootstrap-complete", comment: "some text", data: {}}


--------------------------------------------------------------------------------
/src/test/resources/sql/json/filter.javascript:
--------------------------------------------------------------------------------
 1 | function process_row(rowmap) {
 2 | 	if ( rowmap.data.a ) {
 3 | 		if ( rowmap.data.a.match(/xyzzy/) )
 4 | 			rowmap.suppress();
 5 | 
 6 | 		rowmap.data.a = rowmap.data.a.toUpperCase();
 7 | 	}
 8 | 
 9 | 	if ( rowmap.query && rowmap.query.match(/mangle/) ) {
10 | 		rowmap.query = "mangled";
11 | 	}
12 | 
13 | 	rowmap.partition_string = "test_part_string";
14 | 
15 | 	if ( rowmap.position == "foo" ) {
16 | 		rowmap.query = "crashy crash?";
17 | 	}
18 | }
19 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/filter_rich.javascript:
--------------------------------------------------------------------------------
1 | function process_row(rowmap) {
2 | 	rowmap.data.a = [1,2,3,"b", {"a": "b"}];
3 |   rowmap.data.b = {a: "b"};
4 | }
5 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_bignum:
--------------------------------------------------------------------------------
1 | create database long_db;
2 | use long_db;
3 | create table test_long(id int(10) unsigned not null auto_increment primary key, long_field BIGINT(20) UNSIGNED);
4 | insert into test_long set long_field= 18446744073709551615 ;
5 |   -> {database:"long_db", table: "test_long", type: "insert", data: {"id": 1, long_field: 18446744073709551615} }
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_bit:
--------------------------------------------------------------------------------
 1 | create database bit_db;
 2 | use bit_db;
 3 | create table test_bit(id int(10) unsigned not null auto_increment primary key, bit_field bit(64));
 4 | insert into test_bit set bit_field=b'00011';
 5 |   -> {database:"bit_db", table: "test_bit", type: "insert", data: {"id": 1, bit_field: 3} }
 6 | insert into test_bit set bit_field=b'111111';
 7 |   -> {database:"bit_db", table: "test_bit", type: "insert", data: {"id": 2, bit_field: 63} }
 8 | insert into test_bit set bit_field=409832983;
 9 |   -> {database:"bit_db", table: "test_bit", type: "insert", data: {"id": 3, bit_field: 409832983} }
10 | insert into test_bit set bit_field=18446744073709551615;
11 |   -> {database:"bit_db", table: "test_bit", type: "insert", data: {"id": 4, bit_field: 18446744073709551615} }
12 | insert into test_bit set bit_field=5461;
13 |   -> {database:"bit_db", table: "test_bit", type: "insert", data: {"id": 5, bit_field: 5461} }
14 | 
15 | create table test_bit2 (id int(10) unsigned not null auto_increment primary key, bit_field bit(8));
16 | 
17 | insert into test_bit2 set bit_field=b'00011';
18 |   -> {database:"bit_db", table: "test_bit2", type: "insert", data: {"id": 1, bit_field: 3} }
19 | 
20 | create table test_bit3 (id int(10) unsigned not null auto_increment primary key, bit_field bit(16));
21 | 
22 | insert into test_bit3 set bit_field=13929;
23 |   -> {database:"bit_db", table: "test_bit3", type: "insert", data: {"id": 1, bit_field: 13929} }
24 | 
25 | create table test_bit4 (id int(10) unsigned not null auto_increment primary key, bit_field bit(24));
26 | 
27 | insert into test_bit4 set bit_field=b'111000111000111001010110';
28 |   -> {database:"bit_db", table: "test_bit4", type: "insert", data: {"id": 1, bit_field: 14913110} }
29 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_blob:
--------------------------------------------------------------------------------
 1 | create database blob_db;
 2 | use blob_db;
 3 | create table test_blob(id int(10) unsigned not null auto_increment primary key, blob_field blob);
 4 | insert into test_blob set blob_field=x'00010203';
 5 |   -> {database:"blob_db", table: "test_blob", type: "insert", data: {"id": 1, blob_field: "AAECAw=="} }
 6 | create table test_binary(id int(10) auto_increment primary key, binary_field binary(10));
 7 | insert into test_binary set binary_field=x'00010203';
 8 |   -> {database:"blob_db", table: "test_binary", type: "insert", data: {"id": 1, binary_field: "AAECAw=="} }
 9 | create table test_varbinary(id int(10) auto_increment primary key, varbinary_field varbinary(10));
10 | insert into test_varbinary set varbinary_field=x'00010203';
11 |   -> {database:"blob_db", table: "test_varbinary", type: "insert", data: {"id": 1, varbinary_field: "AAECAw=="} }
12 | 
13 | 
14 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_case_insensitive:
--------------------------------------------------------------------------------
1 | create database case_db;
2 | use case_db;
3 | create table case_db.test_case(ID int(10) unsigned not null auto_increment primary key, BAR_FIELD varchar(255));
4 | insert into test_case set bar_field='bar';
5 |   -> {database:"case_db", table: "test_case", type: "insert", data: {"id": 1, bar_field: "bar"}}
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_charsets:
--------------------------------------------------------------------------------
1 | CREATE DATABASE `test_charsets`
2 | create table `test_charsets`.`c` ( gbk varchar(255) charset 'gbk', big5 varchar(255) charset 'big5', ujis varchar(255) charset 'ujis' )
3 | insert into `test_charsets`.`c` set `gbk` = '乪', `big5` = '圖形碼', `ujis` = '河';
4 | -> {"database": "test_charsets", "table": "c", "type": "insert", "data": {"gbk": "乪", "big5": "圖形碼", "ujis": "河"} }
5 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_column_case:
--------------------------------------------------------------------------------
1 | create database case_columns;
2 | use case_columns;
3 | create table case_columns.test_case(ID int(10) unsigned not null auto_increment primary key, bar_FIELD varchar(255));
4 | insert into test_case set bar_field='bar';
5 |   -> {database:"case_columns", table: "test_case", type: "insert", data: {"ID": 1, "bar_FIELD": "bar"}}
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_create_like:
--------------------------------------------------------------------------------
1 | CREATE DATABASE `test_db`
2 | CREATE TABLE `test_db`.`test_alike` ( id int(11) auto_increment not null primary key, flt float )
3 | CREATE TABLE `test_db`.`test_alike2` LIKE `test_db`.`test_alike`
4 | 
5 | insert into test_db.test_alike2 set flt = 1.5;
6 | -> {"database":"test_db", "table": "test_alike2", "type": "insert", "data": { "id" : 1, "flt": 1.5 } }
7 | 
8 | 
9 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_create_select:
--------------------------------------------------------------------------------
 1 | CREATE DATABASE `test_db_create_select`
 2 | use test_db_create_select
 3 | CREATE TABLE `test_alike` ( id int(11), flt float )
 4 | 
 5 | insert into test_alike set id = 1, flt = 0.5;
 6 | 
 7 | -> {"database":"test_db_create_select", "table": "test_alike", "type": "insert", "data": { "id" : 1, "flt": 0.5 } }
 8 | 
 9 | CREATE TABLE `test_db_create_select`.`test_alike2` select * from test_alike;
10 | 
11 | -> {"database":"test_db_create_select", "table": "test_alike2", "type": "insert", "data": { "id" : 1, "flt": 0.5 } }
12 | 
13 | 
14 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_enum:
--------------------------------------------------------------------------------
 1 | create database enum_db;
 2 | use enum_db;
 3 | create table test_enums(id int(10) unsigned not null auto_increment primary key, enum_field enum('foo', 'bar', 'baz'));
 4 | insert into test_enums set enum_field='bar';
 5 |   -> {database:"enum_db", table: "test_enums", type: "insert", data: {"id": 1, enum_field: "bar"} }
 6 | insert into test_enums set enum_field='baz';
 7 |   -> {database:"enum_db", table: "test_enums", type: "insert", data: {"id": 2, enum_field: "baz"} }
 8 | insert into test_enums set enum_field='foo';
 9 |   -> {database:"enum_db", table: "test_enums", type: "insert", data: {"id": 3, enum_field: "foo"} }
10 | 
11 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_gis:
--------------------------------------------------------------------------------
1 | create database gis_db;
2 | use gis_db;
3 | create table test_gis(id int(10) unsigned not null auto_increment primary key, g GEOMETRY NULL);
4 | insert into test_gis set g=POINT(3, 2);
5 |   -> { database: "gis_db", table: "test_gis", type: "insert", data: { "id": 1, g: "POINT (3 2)"} }
6 | insert into test_gis set g=GEOMETRYCOLLECTION(POINT(1, 1),LINESTRING(POINT(0,1), POINT(1,2), POINT(2,3), POINT(3,4)));
7 |   -> { database: "gis_db", table: "test_gis", type: "insert", data: { "id": 2, g: "GEOMETRYCOLLECTION (POINT (1 1), LINESTRING (0 1, 1 2, 2 3, 3 4))"} }
8 | 
9 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_invalid_time:
--------------------------------------------------------------------------------
 1 | create database timedb_invalid
 2 | use timedb_invalid
 3 | 
 4 | set global time_zone = '-0:00';
 5 | set sql_mode = "ALLOW_INVALID_DATES"
 6 | 
 7 | create table t ( d datetime );
 8 | 
 9 | insert into t values ('1970-01-01 00:00:00');
10 | -> { "database": "timedb_invalid", "table": "t", "type": "insert", "data": { "d": "1970-01-01 00:00:00" } }
11 | insert into t values (0);
12 | -> { "database": "timedb_invalid", "table": "t", "type": "insert", "data": { "d": "0000-00-00 00:00:00" } }
13 | 
14 | CREATE TABLE tt ( t time(6), small_t time(3), dt datetime(6), small_dt datetime(3), ts timestamp(4) NULL, raw_dt datetime, dcol date );
15 | 
16 | insert into tt values (0, 0, 0, 0, 0, 0, 0);
17 | -> { "database": "timedb_invalid", "table": "tt", "type": "insert",
18 |      "data": {
19 |         "t": "00:00:00.000000",
20 |         "small_t": "00:00:00.000",
21 |         "dt": "0000-00-00 00:00:00.000000",
22 |         "small_dt": "0000-00-00 00:00:00.000",
23 |         "ts": "0000-00-00 00:00:00.0000",
24 |         "raw_dt": "0000-00-00 00:00:00",
25 |         "dcol": "0000-00-00"
26 |     } }
27 | 
28 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_javascript_filters:
--------------------------------------------------------------------------------
 1 | use test
 2 | DROP TABLE if exists `test_table`
 3 | CREATE TABLE `test_table` ( id int auto_increment primary key, a text, b text )
 4 | 
 5 | # our javascript filter:
 6 | # 1.  exclude everything that matches "xyzzy"
 7 | # 2.  upper case everything else
 8 | 
 9 | insert into test_table set a='xyzzy'
10 | insert into test_table set a='big big words', b='bvalue2'
11 | 
12 | ->  {"database": "test", "table": "test_table", "type": "insert", "query": "insert into test_table set a='big big words', b='bvalue2'", "data": {"id": 2, "a": "BIG BIG WORDS", "b": "bvalue2"} }
13 | 
14 | insert into test_table set a='mangle'
15 | 
16 | ->  {"database": "test", "table": "test_table", "type": "insert", "query": "mangled", "data": {"id": 3, "a": "MANGLE", "b": null} }
17 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_javascript_filters_rich_values:
--------------------------------------------------------------------------------
 1 | use test
 2 | DROP TABLE if exists `test_table_z`
 3 | CREATE TABLE `test_table_z` ( id int auto_increment primary key, a text, b text )
 4 | 
 5 | # our javascript filter:
 6 | # 1.  exclude everything that matches "xyzzy"
 7 | # 2.  upper case everything else
 8 | 
 9 | insert into test_table_z set a='xyzzy'
10 | 
11 | ->  {"database": "test", "table": "test_table_z", "type": "insert", "data": {"id": 1, "a":  [1,2,3,"b", {"a": "b"}], "b": {"a": "b"}} }
12 | 
13 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_json:
--------------------------------------------------------------------------------
 1 | CREATE DATABASE `test_json`
 2 | create table `test_json`.`json` ( j json, i int )
 3 | use test_json
 4 | insert into `json` set `j` = '{"foo": "bar", "baz": [1,2,3,{"hello": "inside an array"}], "bar": 123.4321}', i = 0;
 5 | -> {"database": "test_json", "table": "json", "type": "insert", "data": {"j": {"foo": "bar", "baz": [1,2,3,{"hello": "inside an array"}], "bar": 123.4321}, i: 0}}
 6 | 
 7 | update `json` set i = 1;
 8 | 
 9 | -> {"database": "test_json", "table": "json", "type": "update", "data": {i: 1, j: {"foo": "bar", "baz": [1,2,3,{"hello": "inside an array"}], "bar": 123.4321}}, old: {i: 0}}
10 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_latin1:
--------------------------------------------------------------------------------
1 | CREATE DATABASE `test_latin1` default charset 'latin1'
2 | create table `test_latin1`.`latin1` ( textcol text )
3 | use test_latin1
4 | insert into `latin1` set `textcol` = 'áááááááá';
5 | -> {"database": "test_latin1", "table": "latin1", "type": "insert", "data": {"textcol": "áááááááá"} }
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_minimal:
--------------------------------------------------------------------------------
 1 | use test
 2 | DROP TABLE if exists `test_table`
 3 | CREATE TABLE `test_table` ( id int auto_increment primary key, a text, b text )
 4 | insert into test_table set a='avalue'
 5 | 
 6 | ->  {"database": "test", "table": "test_table", "type": "insert", "data": {"id": 1, "a": "avalue"} }
 7 | 
 8 | insert into test_table set a='avalue2', b='bvalue2'
 9 | 
10 | ->  {"database": "test", "table": "test_table", "type": "insert", "data": {"id": 2, "a": "avalue2", "b": "bvalue2"} }
11 | 
12 | update test_table set a='avalue3' where id = 2
13 | 
14 | ->  { "database": "test", "table": "test_table", "type": "update", "data": {"id": 2, "a": "avalue3"} }
15 | 
16 | 
17 | # testing an open-replicator bug around >1 byte of NULLability columns
18 | CREATE TABLE `test_table_2` ( id int auto_increment primary key, i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int, i10 int );
19 | 
20 | insert into test_table_2 set i1=555, i2=777;
21 | ->  { database: "test", table: "test_table_2", type: "insert", data: { id: 1, i1: 555, i2: 777 } }
22 | 
23 | update test_table_2 set i3=444 where i1=555;
24 | ->  { database: "test", table: "test_table_2", type: "update", data: { id: 1, i3: 444 } }
25 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_row_query_log_is_on:
--------------------------------------------------------------------------------
 1 | CREATE DATABASE `test_row_query`;
 2 | CREATE TABLE `test_row_query`.`row_query` ( id int );
 3 | 
 4 | insert into `test_row_query`.`row_query` set id = 10;
 5 |     -> { "database": "test_row_query", "table": "row_query", "query" : "insert into `test_row_query`.`row_query` set id = 10", "type": "insert", "data": { "id": 10 } }
 6 | 
 7 | insert into `test_row_query`.`row_query` set id = 20 /* comment:insert */
 8 |     -> { "database": "test_row_query", "table": "row_query", "query" : "insert into `test_row_query`.`row_query` set id = 20 /* comment:insert */", "type": "insert", "data": { "id": 20 } }
 9 | 
10 | update `test_row_query`.`row_query` set id = 11 where id = 10
11 |     -> { "database": "test_row_query", "table": "row_query", "query" : "update `test_row_query`.`row_query` set id = 11 where id = 10", "type": "update", "data": { "id": 11 }, old: { "id": 10} }
12 | 
13 | update `test_row_query`.`row_query` set id = 21 where id = 20 /* comment:update */
14 |     -> { "database": "test_row_query", "table": "row_query", "query" : "update `test_row_query`.`row_query` set id = 21 where id = 20 /* comment:update */", "type": "update", "data": { "id": 21 }, old: { "id": 20} }
15 | 
16 | delete from `test_row_query`.`row_query` where id = 11
17 |     -> { "database": "test_row_query", "table": "row_query", "query" : "delete from `test_row_query`.`row_query` where id = 11", "type": "delete", "data": { "id": 11 } }
18 | 
19 | delete from `test_row_query`.`row_query` where id = 21 /* comment:delete */
20 |     -> { "database": "test_row_query", "table": "row_query", "query" : "delete from `test_row_query`.`row_query` where id = 21 /* comment:delete */", "type": "delete", "data": { "id": 21 } }
21 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_set:
--------------------------------------------------------------------------------
 1 | create database set_db;
 2 | use set_db;
 3 | create table test_sets(id int(10) unsigned not null auto_increment primary key, set_field set('foo', 'bar', 'baz') NOT NULL);
 4 | insert into test_sets set set_field='bar,baz';
 5 |   -> {database:"set_db", table: "test_sets", type: "insert", data: {"id": 1, set_field: ["bar", "baz"]} }
 6 | insert into test_sets set set_field='baz,foo';
 7 |   -> {database:"set_db", table: "test_sets", type: "insert", data: {"id": 2, set_field: ["foo", "baz"]} }
 8 | insert into test_sets set set_field='foo';
 9 |   -> {database:"set_db", table: "test_sets", type: "insert", data: {"id": 3, set_field: ["foo"]} }
10 | insert into test_sets set set_field='';
11 |   -> {database:"set_db", table: "test_sets", type: "insert", data: {"id": 4, set_field: []} }
12 | alter table test_sets modify column set_field set('foo', 'bar', 'baz') NULL DEFAULT NULL;
13 | insert into test_sets set id=NULL;
14 |   -> {database:"set_db", table: "test_sets", type: "insert", data: {"id": 5, set_field: null} }
15 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_time:
--------------------------------------------------------------------------------
 1 | create database timedb
 2 | use timedb
 3 | 
 4 | set global time_zone = '-0:00';
 5 | 
 6 | CREATE TABLE tt ( t time(6), small_t time(3), dt datetime(6), small_dt datetime(3), ts timestamp(4) NULL );
 7 | 
 8 | insert into tt set dt = '0001-01-01 00-00-00';
 9 | -> { "database": "timedb", "table": "tt", "type": "insert", "data": {"t": null, "small_t": null, "dt": "0001-01-01 00:00:00.000000", "small_dt": null, "ts": null } }
10 | 
11 | insert into tt set t = '12:23:33.123456', small_t = '12:23:33.345', dt = '2012-01-01 22:32:34.222222', small_dt = '2012-01-01 22:32:34.123', ts = '2012-01-01 22:32:34.1234';
12 | -> { "database": "timedb", "table": "tt", "type": "insert", "data": {"t": "12:23:33.123456", "small_t": "12:23:33.345", "dt": "2012-01-01 22:32:34.222222", "small_dt": "2012-01-01 22:32:34.123", "ts": "2012-01-01 22:32:34.1234" } }
13 | 
14 | insert into tt (dt) values('2000-01-01 01:00:00.999999');
15 | -> { "database": "timedb", "table": "tt", "type": "insert", "data": {"t": null, "small_t": null, "dt": "2000-01-01 01:00:00.999999", "small_dt": null, "ts": null } }
16 | 
17 | insert into tt (dt) values('1950-01-01 01:00:00.999999');
18 | -> { "database": "timedb", "table": "tt", "type": "insert", "data": {"t": null, "small_t": null, "dt": "1950-01-01 01:00:00.999999", "small_dt": null, "ts": null } }
19 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_ucs2:
--------------------------------------------------------------------------------
1 | CREATE DATABASE `test_ucs2` default charset 'ucs2'
2 | create table `test_ucs2`.`ucs2` ( textcol text )
3 | insert into `test_ucs2`.`ucs2` set `textcol` = '∆˚ø¨´˚∆ƒ¬ß∂µ≤˜π∑ˆ¨´øßå¬∂˚∆ƒ…¬å˚∆ß';
4 | -> {"database": "test_ucs2", "table": "ucs2", "type": "insert", "data": {"textcol": "∆˚ø¨´˚∆ƒ¬ß∂µ≤˜π∑ˆ¨´øßå¬∂˚∆ƒ…¬å˚∆ß"} }
5 | 
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/json/test_zero_created_at:
--------------------------------------------------------------------------------
1 | CREATE DATABASE `cdb`
2 | create table `cdb`.`ca` ( id int(11) auto_increment not null primary key, datecol datetime NOT NULL default 0, ch varchar(255));
3 | insert into cdb.ca set ch='a';
4 | -> {"database":"cdb", "table": "ca", "type": "insert", "data":  {"id": 1, "ch": "a", "datecol": "0000-00-00 00:00:00"}  }
5 | insert into cdb.ca set ch='a', datecol = '0003-00-00 00:00:01';
6 | -> {"database":"cdb", "table": "ca", "type": "insert", "data":  {"id": 2, "ch": "a", "datecol": "0000-00-00 00:00:00"}  }
7 | 
8 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/rows/rows.sql:
--------------------------------------------------------------------------------
1 | INSERT into sharded set account_id = 1, nice_id = 1, status_id = 2, date_field = '1979-10-10', text_field = 'Some Text', latin1_field = 'FooBarä', utf8_field = 'FooBarä', float_field = 1.333333333333, timestamp_field = '1980-01-01', decimal_field = 8.621
2 | INSERT into sharded set account_id = 1, nice_id = 2, status_id = 2, date_field = '1979-10-10', text_field = 'Delete Me', latin1_field = 'FooBarä', utf8_field = 'FooBarä', float_field = 1.333333333333, timestamp_field = '1980-01-01', decimal_field = 8.621
3 | FLUSH LOGS
4 | UPDATE sharded set status_id = 1, text_field = 'Updated Text', timestamp_field=timestamp_field where id = 1
5 | DELETE FROM sharded where nice_id = 2
6 | 
7 | DROP DATABASE IF EXISTS `shard_2`
8 | CREATE DATABASE `shard_2`
9 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/enum.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE IF NOT EXISTS `test`.`enum_test` (
2 |   `language` ENUM('en-US', 'de-DE'),
3 |   `decimal_separator` ENUM(',', '.'),
4 |   PRIMARY KEY (`language`)
5 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
6 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/mediumint.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE `mediumints` (
2 |   `id` bigint(20) NOT NULL AUTO_INCREMENT,
3 |   `account_id` int(11) NOT NULL,
4 |   `medium` mediumint,
5 |   PRIMARY KEY (id)
6 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8
7 | 
8 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/minimal.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE `minimal` (
2 |   `id` bigint(20) NOT NULL AUTO_INCREMENT,
3 |   `account_id` int(11) NOT NULL,
4 |   `text_field` varchar(96),
5 |   PRIMARY KEY (id, text_field)
6 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8
7 | 
8 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/sharded.sql:
--------------------------------------------------------------------------------
 1 | CREATE TABLE `sharded` (
 2 |   `id` bigint(20) NOT NULL AUTO_INCREMENT,
 3 |   `account_id` int(11) UNSIGNED NOT NULL,
 4 |   `nice_id` int(11) NOT NULL,
 5 |   `status_id` tinyint NOT NULL default 2,
 6 |   `date_field` datetime,
 7 |   `text_field` text,
 8 |   `latin1_field` varchar(96) CHARACTER SET latin1 NOT NULL DEFAULT '',
 9 |   `utf8_field` varchar(96) CHARACTER SET utf8 NOT NULL DEFAULT '',
10 |   `float_field` float(5,2),
11 |   `timestamp_field` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
12 |   `timestamp2_field` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
13 |   `datetime2_field` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
14 |   -- Can't have a default time
15 |   -- See http://dev.mysql.com/doc/refman/5.6/en/timestamp-initialization.html
16 |   `time2_field` time(6),
17 |   `decimal_field` decimal(12,7),
18 |   PRIMARY KEY (`id`, `account_id`)
19 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8
20 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/sharded_55.sql:
--------------------------------------------------------------------------------
 1 | CREATE TABLE `sharded` (
 2 |   `id` bigint(20) NOT NULL AUTO_INCREMENT,
 3 |   `account_id` int(11) UNSIGNED NOT NULL,
 4 |   `nice_id` int(11) NOT NULL,
 5 |   `status_id` tinyint NOT NULL default 2,
 6 |   `date_field` datetime,
 7 |   `text_field` text,
 8 |   `latin1_field` varchar(96) CHARACTER SET latin1 NOT NULL DEFAULT '',
 9 |   `utf8_field` varchar(96) CHARACTER SET utf8 NOT NULL DEFAULT '',
10 |   `float_field` float(5,2),
11 |   `timestamp_field` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
12 |   `decimal_field` decimal(12,7),
13 |   PRIMARY KEY (`id`, `account_id`)
14 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8
15 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/schema/unsigned.sql:
--------------------------------------------------------------------------------
 1 | CREATE TABLE `ints` (
 2 |   `id` bigint(20) NOT NULL AUTO_INCREMENT,
 3 |   `i64` bigint UNSIGNED,
 4 |   `i32` int UNSIGNED,
 5 |   `i24` mediumint UNSIGNED,
 6 |   `i16` smallint UNSIGNED,
 7 |   `i8`  tinyint  UNSIGNED,
 8 |   PRIMARY KEY (id)
 9 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8
10 | 
11 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/serialization/alter_table:
--------------------------------------------------------------------------------
 1 | CREATE database `ser3`;
 2 | CREATE table `ser3`.`tbl` ( id mediumint, `a` varchar(255) character set latin1, `b` mediumtext, `c` enum('a','b'), primary KEY(id) );
 3 | ALTER table `ser3`.`tbl` add column `d` blob not null after `b`;
 4 | -> { type: "table-alter",
 5 |      database: "ser3",
 6 |      table: "tbl",
 7 |      old: {
 8 |        database: "ser3",
 9 |        table: "tbl",
10 |        charset: "utf8",
11 |        primary-key: [ "id" ],
12 |        columns: [
13 |         { name: "id", type: "mediumint",  signed: true },
14 |         { name: "a",  type: "varchar",    charset: "latin1" },
15 |         { name: "b",  type: "mediumtext", charset: "utf8" },
16 |         { name: "c",  type: "enum",       enum-values: [ "a", "b" ] }
17 |        ]
18 |      },
19 |      def: {
20 |        database: "ser3",
21 |        table: "tbl",
22 |        charset: "utf8",
23 |        primary-key: [ "id" ],
24 |        columns: [
25 |         { name: "id", type: "mediumint",  signed: true },
26 |         { name: "a",  type: "varchar",    charset: "latin1" },
27 |         { name: "b",  type: "mediumtext", charset: "utf8" },
28 |         { name: "d",  type: "blob",       charset: "binary" },
29 |         { name: "c",  type: "enum",       enum-values: [ "a", "b" ] }
30 |        ]
31 |      }
32 |    }
33 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/serialization/alter_table_utf8mb3:
--------------------------------------------------------------------------------
 1 | CREATE database `ser3`;
 2 | CREATE table `ser3`.`tbl` ( id mediumint, `a` varchar(255) character set latin1, `b` mediumtext, `c` enum('a','b'), primary KEY(id) );
 3 | ALTER table `ser3`.`tbl` add column `d` blob not null after `b`;
 4 | -> { type: "table-alter",
 5 |      database: "ser3",
 6 |      table: "tbl",
 7 |      old: {
 8 |        database: "ser3",
 9 |        table: "tbl",
10 |        charset: "utf8mb3",
11 |        primary-key: [ "id" ],
12 |        columns: [
13 |         { name: "id", type: "mediumint",  signed: true },
14 |         { name: "a",  type: "varchar",    charset: "latin1" },
15 |         { name: "b",  type: "mediumtext", charset: "utf8mb3" },
16 |         { name: "c",  type: "enum",       enum-values: [ "a", "b" ] }
17 |        ]
18 |      },
19 |      def: {
20 |        database: "ser3",
21 |        table: "tbl",
22 |        charset: "utf8mb3",
23 |        primary-key: [ "id" ],
24 |        columns: [
25 |         { name: "id", type: "mediumint",  signed: true },
26 |         { name: "a",  type: "varchar",    charset: "latin1" },
27 |         { name: "b",  type: "mediumtext", charset: "utf8mb3" },
28 |         { name: "d",  type: "blob",       charset: "binary" },
29 |         { name: "c",  type: "enum",       enum-values: [ "a", "b" ] }
30 |        ]
31 |      }
32 |    }
33 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/serialization/create_database:
--------------------------------------------------------------------------------
 1 | CREATE database `ser_d1` charset 'latin1';
 2 | -> { type: "database-create", database: "ser_d1", charset: "latin1" }
 3 | 
 4 | CREATE database if not exists `ser_d2`;
 5 | -> { type: "database-create", database: "ser_d2", charset: "utf8" }
 6 | 
 7 | ALTER database `ser_d1` charset 'utf8'
 8 | -> { type: "database-alter", database: "ser_d1", charset: "utf8" }
 9 | 
10 | DROP database `ser_d2`;
11 | -> { type: "database-drop", database: "ser_d2" }
12 | 
13 | DROP database if exists `ser_d1`;
14 | -> { type: "database-drop", database: "ser_d1" }
15 | 
16 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/serialization/create_table_55:
--------------------------------------------------------------------------------
 1 | CREATE database `ser1`;
 2 | -> { type: "database-create", database: "ser1", charset: "utf8" }
 3 | CREATE table `ser1`.`tbl1` ( id mediumint, `a` varchar(255) character set latin1, `b` mediumtext, `c` enum('a','b'), primary KEY(id) );
 4 | -> {
 5 |      type: "table-create",
 6 |      database: "ser1",
 7 |      table: "tbl1",
 8 |      def: {
 9 |        database: "ser1",
10 |        table: "tbl1",
11 |        columns: [
12 |                { name: "id", type: "mediumint", signed: true },
13 |                { name: "a",  type: "varchar", charset: "latin1" },
14 |                { name: "b",  type: "mediumtext", charset: "utf8" },
15 |                { name: "c",  type: "enum", enum-values: ["a", "b"] }
16 |        ],
17 |        primary-key: [ "id" ],
18 |        charset: "utf8"
19 |      }
20 |    }
21 | 
22 | CREATE table `ser1`.`tbl2` ( id int unsigned, `a` BIT, primary key (id, a));
23 | ->{
24 |     type: "table-create",
25 |     database: "ser1",
26 |     table: "tbl2",
27 |     def: {
28 |       database: "ser1",
29 |       table: "tbl2",
30 |       columns: [
31 |         { name: "id", type: "int", signed: false },
32 |         { name: "a", type: "bit" }
33 |       ],
34 |       primary-key: [ "id", "a" ],
35 |       charset: "utf8"
36 |     }
37 |   }
38 | 
39 | CREATE database `ser1like`;
40 | CREATE table `ser1like`.`tbllike` LIKE `ser1`.`tbl2`;
41 | ->{
42 |     type: "table-create",
43 |     database: "ser1like",
44 |     table: "tbllike",
45 |     def: {
46 |       database: "ser1like",
47 |       table: "tbllike",
48 |       columns: [
49 |         { name: "id", type: "int", signed: false },
50 |         { name: "a", type: "bit" }
51 |       ],
52 |       primary-key: [ "id", "a" ],
53 |       charset: "utf8"
54 |     }
55 |   }
56 | 
57 | 


--------------------------------------------------------------------------------
/src/test/resources/sql/serialization/drop_table:
--------------------------------------------------------------------------------
 1 | CREATE database `serdrop`;
 2 | CREATE table `serdrop`.`tbl` ( id int );
 3 | DROP table `serdrop`.`tbl`;
 4 | 
 5 | -> {
 6 |      type: "table-drop",
 7 |      database: "serdrop",
 8 |      table: "tbl"
 9 |    }
10 | 
11 | 


--------------------------------------------------------------------------------
/src/test/travis.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | MAVEN_STUPID_PATTERN="Downloading:|Downloaded:"
 4 | set -eux
 5 | echo travis_fold:start:compile
 6 | KAFKA_VERSION=0.8.2.2 make depclean compile | grep -vE "$MAVEN_STUPID_PATTERN"
 7 | KAFKA_VERSION=0.9.0.1 make depclean compile | grep -vE "$MAVEN_STUPID_PATTERN"
 8 | KAFKA_VERSION=0.10.0.1 make depclean compile | grep -vE "$MAVEN_STUPID_PATTERN"
 9 | KAFKA_VERSION=0.10.2.1 make depclean compile | grep -vE "$MAVEN_STUPID_PATTERN"
10 | KAFKA_VERSION=0.11.0.1 make depclean compile | grep -vE "$MAVEN_STUPID_PATTERN"
11 | make depclean 2>&1 | grep -vE "$MAVEN_STUPID_PATTERN"
12 | echo travis_fold:end:compile
13 | make test 2>&1 | grep -vE "$MAVEN_STUPID_PATTERN"
14 | exit ${PIPESTATUS[0]}
15 | 


--------------------------------------------------------------------------------