├── .gitignore
├── .github
    └── workflows
    │   └── release.yml
├── README.md
├── pom.xml
└── src
    └── main
        └── java
            └── com
                └── darcytech
                    └── debezium
                        └── converter
                            └── MySqlDateTimeConverter.java


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | .idea/
3 | .vscode/
4 | .settings/
5 | target/
6 | .classpath
7 | .factorypath
8 | .project


--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
 1 | # This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
 2 | # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
 3 | 
 4 | name: Maven Package
 5 | 
 6 | on:
 7 |   workflow_dispatch:
 8 |     inputs:
 9 |       version:
10 |         description: "Release version"
11 |         required: true
12 |         default: "1.4.0.Final"
13 | 
14 | jobs:
15 |   release:
16 | 
17 |     runs-on: ubuntu-latest
18 | 
19 |     steps:
20 |       - uses: actions/checkout@v2
21 |       - name: Set up JDK 11
22 |         uses: actions/setup-java@v2
23 |         with:
24 |           java-version: '11'
25 |           distribution: 'adopt'
26 |           server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
27 |           settings-path: ${{ github.workspace }} # location for the settings.xml file
28 | 
29 |       - name: Build with Maven
30 |         run: mvn -B package --file pom.xml
31 | 
32 |       - uses: "marvinpinto/action-automatic-releases@latest"
33 |         with:
34 |           repo_token: "${{ secrets.GITHUB_TOKEN }}"
35 |           automatic_release_tag: "v${{ github.event.inputs.version }}"
36 |           prerelease: false
37 |           files: |
38 |             LICENSE.txt
39 |             **/target/debezium-extension*.jar


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | **Note: debezium has been upgraded and the repository is no longer maintained. Please see the [latest official documentation](https://debezium.io/releases/)**
 2 | 
 3 | [![Debezium architecture](https://debezium.io/documentation/reference/1.4/_images/debezium-architecture.png)](https://debezium.io/documentation/reference/1.4/connectors/mysql.html)
 4 | 
 5 | [![Build Status(https://github.com/holmofy/debezium-datetime-converter/actions/workflows/release.yml/badge.svg)](https://github.com/holmofy/debezium-datetime-converter/actions/workflows/release.yml/badge.svg)](https://github.com/holmofy/debezium-datetime-converter/releases)
 6 | 
 7 | # debezium-datetime-converter
 8 | 
 9 | Debezium [custom converter](https://debezium.io/documentation/reference/development/converters.html) is used to deal with mysql [datetime type problems](https://debezium.io/documentation/reference/1.5/connectors/mysql.html#mysql-temporal-types).
10 | 
11 | | mysql                               | binlog-connector                         | debezium                          | schema                 |
12 | | ----------------------------------- | ---------------------------------------- | --------------------------------- | ----------------------------------- |
13 | | date<br>(2021-01-28)                | LocalDate<br/>(2021-01-28)               | Integer<br/>(18655)               | io.debezium.time.Date               |
14 | | time<br/>(17:29:04)                 | Duration<br/>(PT17H29M4S)                | Long<br/>(62944000000)            | io.debezium.time.Time               |
15 | | timestamp<br/>(2021-01-28 17:29:04) | ZonedDateTime<br/>(2021-01-28T09:29:04Z) | String<br/>(2021-01-28T09:29:04Z) | io.debezium.time.ZonedTimestamp     |
16 | | Datetime<br/>(2021-01-28 17:29:04)  | LocalDateTime<br/>(2021-01-28T17:29:04)  | Long<br/>(1611854944000)          | io.debezium.time.Timestamp          |
17 | 
18 | > For details, please refer to [this article](https://blog.hufeifei.cn/2021/03/DB/mysql-binlog-parser/)
19 | 
20 | # Usage
21 | 
22 | 1. [Download](https://github.com/holmofy/debezium-datetime-converter/releases) the extended jar package and put it in the same level directory of the debezium plugin.
23 | 
24 | 2. In debezium-connector, Add the following configuration: 
25 | 
26 | ```properties
27 | connector.class=io.debezium.connector.mysql.MySqlConnector
28 | # ...
29 | converters=datetime
30 | datetime.type=com.darcytech.debezium.converter.MySqlDateTimeConverter
31 | datetime.format.date=yyyy-MM-dd
32 | datetime.format.time=HH:mm:ss
33 | datetime.format.datetime=yyyy-MM-dd HH:mm:ss
34 | datetime.format.timestamp=yyyy-MM-dd HH:mm:ss
35 | datetime.format.timestamp.zone=UTC+8
36 | ```
37 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 |     <!--<parent>-->
 7 |         <!--<groupId>io.debezium</groupId>-->
 8 |         <!--<artifactId>debezium-parent</artifactId>-->
 9 |         <!--<version>1.4.0.Final</version>-->
10 |     <!--</parent>-->
11 |     <groupId>com.darcytech.debezium</groupId>
12 |     <artifactId>debezium-extension</artifactId>
13 |     <version>1.4.0.Final</version>
14 |     <packaging>jar</packaging>
15 | 
16 |     <properties>
17 |         <version.debezium>1.4.0.Final</version.debezium>
18 |         <version.kafka>2.6.0</version.kafka>
19 |     </properties>
20 |     <dependencies>
21 |         <dependency>
22 |             <groupId>io.debezium</groupId>
23 |             <artifactId>debezium-api</artifactId>
24 |             <version>${version.debezium}</version>
25 |             <scope>provided</scope>
26 |         </dependency>
27 |         <dependency>
28 |             <groupId>io.debezium</groupId>
29 |             <artifactId>debezium-connector-mysql</artifactId>
30 |             <version>${version.debezium}</version>
31 |             <scope>provided</scope>
32 |         </dependency>
33 |         <dependency>
34 |             <groupId>org.apache.kafka</groupId>
35 |             <artifactId>connect-api</artifactId>
36 |             <version>${version.kafka}</version>
37 |             <scope>provided</scope>
38 |         </dependency>
39 |         <dependency>
40 |             <groupId>org.projectlombok</groupId>
41 |             <artifactId>lombok</artifactId>
42 |             <version>1.18.16</version>
43 |             <scope>provided</scope>
44 |         </dependency>
45 |     </dependencies>
46 | 
47 |     <build>
48 |         <plugins>
49 |             <plugin>
50 |                 <groupId>org.apache.maven.plugins</groupId>
51 |                 <artifactId>maven-compiler-plugin</artifactId>
52 |                 <version>3.8.1</version>
53 |                 <configuration>
54 |                     <source>1.8</source>
55 |                     <target>1.8</target>
56 |                     <annotationProcessorPaths>
57 |                         <path>
58 |                             <groupId>org.projectlombok</groupId>
59 |                             <artifactId>lombok</artifactId>
60 |                             <version>1.18.16</version>
61 |                         </path>
62 |                     </annotationProcessorPaths>
63 |                 </configuration>
64 |             </plugin>
65 |         </plugins>
66 |     </build>
67 | 
68 | </project>


--------------------------------------------------------------------------------
/src/main/java/com/darcytech/debezium/converter/MySqlDateTimeConverter.java:
--------------------------------------------------------------------------------
  1 | package com.darcytech.debezium.converter;
  2 | 
  3 | import io.debezium.spi.converter.CustomConverter;
  4 | import io.debezium.spi.converter.RelationalColumn;
  5 | import lombok.extern.slf4j.Slf4j;
  6 | import org.apache.kafka.connect.data.SchemaBuilder;
  7 | 
  8 | import java.time.*;
  9 | import java.time.format.DateTimeFormatter;
 10 | import java.util.Properties;
 11 | import java.util.function.Consumer;
 12 | 
 13 | /**
 14 |  * 处理Debezium时间转换的问题
 15 |  * Debezium默认将MySQL中datetime类型转成UTC的时间戳({@link io.debezium.time.Timestamp})，时区是写死的没法儿改，
 16 |  * 导致数据库中设置的UTC+8，到kafka中变成了多八个小时的long型时间戳
 17 |  * Debezium默认将MySQL中的timestamp类型转成UTC的字符串。
 18 |  * | mysql                               | mysql-binlog-connector                   | debezium                          |
 19 |  * | ----------------------------------- | ---------------------------------------- | --------------------------------- |
 20 |  * | date<br>(2021-01-28)                | LocalDate<br/>(2021-01-28)               | Integer<br/>(18655)               |
 21 |  * | time<br/>(17:29:04)                 | Duration<br/>(PT17H29M4S)                | Long<br/>(62944000000)            |
 22 |  * | timestamp<br/>(2021-01-28 17:29:04) | ZonedDateTime<br/>(2021-01-28T09:29:04Z) | String<br/>(2021-01-28T09:29:04Z) |
 23 |  * | Datetime<br/>(2021-01-28 17:29:04)  | LocalDateTime<br/>(2021-01-28T17:29:04)  | Long<br/>(1611854944000)          |
 24 |  *
 25 |  * @see io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter
 26 |  */
 27 | @Slf4j
 28 | public class MySqlDateTimeConverter implements CustomConverter<SchemaBuilder, RelationalColumn> {
 29 | 
 30 |     private DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_DATE;
 31 |     private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_TIME;
 32 |     private DateTimeFormatter datetimeFormatter = DateTimeFormatter.ISO_DATE_TIME;
 33 |     private DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_DATE_TIME;
 34 | 
 35 |     private ZoneId timestampZoneId = ZoneId.systemDefault();
 36 | 
 37 |     @Override
 38 |     public void configure(Properties props) {
 39 |         readProps(props, "format.date", p -> dateFormatter = DateTimeFormatter.ofPattern(p));
 40 |         readProps(props, "format.time", p -> timeFormatter = DateTimeFormatter.ofPattern(p));
 41 |         readProps(props, "format.datetime", p -> datetimeFormatter = DateTimeFormatter.ofPattern(p));
 42 |         readProps(props, "format.timestamp", p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
 43 |         readProps(props, "format.timestamp.zone", z -> timestampZoneId = ZoneId.of(z));
 44 |     }
 45 | 
 46 |     private void readProps(Properties properties, String settingKey, Consumer<String> callback) {
 47 |         String settingValue = (String) properties.get(settingKey);
 48 |         if (settingValue == null || settingValue.length() == 0) {
 49 |             return;
 50 |         }
 51 |         try {
 52 |             callback.accept(settingValue.trim());
 53 |         } catch (IllegalArgumentException | DateTimeException e) {
 54 |             log.error("The \"{}\" setting is illegal:{}", settingKey, settingValue);
 55 |             throw e;
 56 |         }
 57 |     }
 58 | 
 59 |     @Override
 60 |     public void converterFor(RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
 61 |         String sqlType = column.typeName().toUpperCase();
 62 |         SchemaBuilder schemaBuilder = null;
 63 |         Converter converter = null;
 64 |         if ("DATE".equals(sqlType)) {
 65 |             schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.date.string");
 66 |             converter = this::convertDate;
 67 |         }
 68 |         if ("TIME".equals(sqlType)) {
 69 |             schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.time.string");
 70 |             converter = this::convertTime;
 71 |         }
 72 |         if ("DATETIME".equals(sqlType)) {
 73 |             schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.datetime.string");
 74 |             converter = this::convertDateTime;
 75 |         }
 76 |         if ("TIMESTAMP".equals(sqlType)) {
 77 |             schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.timestamp.string");
 78 |             converter = this::convertTimestamp;
 79 |         }
 80 |         if (schemaBuilder != null) {
 81 |             registration.register(schemaBuilder, converter);
 82 |             log.info("register converter for sqlType {} to schema {}", sqlType, schemaBuilder.name());
 83 |         }
 84 |     }
 85 | 
 86 |     private String convertDate(Object input) {
 87 |         if (input instanceof LocalDate) {
 88 |             return dateFormatter.format((LocalDate) input);
 89 |         }
 90 |         if (input instanceof Integer) {
 91 |             LocalDate date = LocalDate.ofEpochDay((Integer) input);
 92 |             return dateFormatter.format(date);
 93 |         }
 94 |         return null;
 95 |     }
 96 | 
 97 |     private String convertTime(Object input) {
 98 |         if (input instanceof Duration) {
 99 |             Duration duration = (Duration) input;
100 |             long seconds = duration.getSeconds();
101 |             int nano = duration.getNano();
102 |             LocalTime time = LocalTime.ofSecondOfDay(seconds).withNano(nano);
103 |             return timeFormatter.format(time);
104 |         }
105 |         return null;
106 |     }
107 | 
108 |     private String convertDateTime(Object input) {
109 |         if (input instanceof LocalDateTime) {
110 |             return datetimeFormatter.format((LocalDateTime) input);
111 |         }
112 |         return null;
113 |     }
114 | 
115 |     private String convertTimestamp(Object input) {
116 |         if (input instanceof ZonedDateTime) {
117 |             // mysql的timestamp会转成UTC存储，这里的zonedDatetime都是UTC时间
118 |             ZonedDateTime zonedDateTime = (ZonedDateTime) input;
119 |             LocalDateTime localDateTime = zonedDateTime.withZoneSameInstant(timestampZoneId).toLocalDateTime();
120 |             return timestampFormatter.format(localDateTime);
121 |         }
122 |         return null;
123 |     }
124 | 
125 | }
126 | 


--------------------------------------------------------------------------------