├── README.md └── flink-cdc-iceberg-demo ├── .DS_Store ├── docker-compose.yml └── sql-client ├── .DS_Store ├── Dockerfile ├── bin └── sql-client ├── conf ├── flink-conf.yaml └── sql-client-conf.yaml ├── docker-entrypoint.sh └── lib ├── .DS_Store ├── flink-shaded-hadoop-2-uber-2.7.5-10.0.jar ├── flink-sql-connector-mysql-cdc-2.1-SNAPSHOT.jar └── iceberg-flink-1.13-runtime-0.13.0-SNAPSHOT.jar /README.md: -------------------------------------------------------------------------------- 1 | # flink-cdc-tutorial 2 | -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/.DS_Store -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | sql-client: 4 | user: flink:flink 5 | image: yuxialuo/flink-sql-client:1.13.2.v1 6 | depends_on: 7 | - jobmanager 8 | - mysql 9 | environment: 10 | FLINK_JOBMANAGER_HOST: jobmanager 11 | MYSQL_HOST: mysql 12 | volumes: 13 | - shared-tmpfs:/tmp/iceberg 14 | jobmanager: 15 | user: flink:flink 16 | image: flink:1.13.2-scala_2.11 17 | ports: 18 | - "8081:8081" 19 | command: jobmanager 20 | environment: 21 | - | 22 | FLINK_PROPERTIES= 23 | jobmanager.rpc.address: jobmanager 24 | volumes: 25 | - shared-tmpfs:/tmp/iceberg 26 | taskmanager: 27 | user: flink:flink 28 | image: flink:1.13.2-scala_2.11 29 | depends_on: 30 | - jobmanager 31 | command: taskmanager 32 | environment: 33 | - | 34 | FLINK_PROPERTIES= 35 | jobmanager.rpc.address: jobmanager 36 | taskmanager.numberOfTaskSlots: 2 37 | volumes: 38 | - shared-tmpfs:/tmp/iceberg 39 | mysql: 40 | image: debezium/example-mysql:1.1 41 | ports: 42 | - "3306:3306" 43 | environment: 44 | - MYSQL_ROOT_PASSWORD=123456 45 | - MYSQL_USER=mysqluser 46 | - MYSQL_PASSWORD=mysqlpw 47 | 48 | volumes: 49 | shared-tmpfs: 50 | driver: local 51 | driver_opts: 52 | type: "tmpfs" 53 | device: "tmpfs" 54 | -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/sql-client/.DS_Store -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM maven:3.6-jdk-8-slim AS builder 2 | 3 | FROM flink:1.13.2-scala_2.11 4 | 5 | RUN apt-get update 6 | RUN apt install tree 7 | 8 | # Copy sql-client script 9 | COPY bin/* /opt/sql-client/ 10 | RUN mkdir -p /opt/sql-client/lib 11 | 12 | # Copy configuration 13 | COPY conf/* /opt/flink/conf/ 14 | 15 | WORKDIR /opt/sql-client 16 | ENV SQL_CLIENT_HOME /opt/sql-client 17 | 18 | 19 | COPY docker-entrypoint.sh / 20 | 21 | COPY lib/* /opt/sql-client/lib/ 22 | COPY lib/* /opt/flink/lib/ 23 | 24 | RUN ["chmod", "+x", "/docker-entrypoint.sh"] 25 | RUN ["chmod", "+x", "/opt/sql-client/sql-client"] 26 | ENTRYPOINT ["/docker-entrypoint.sh"] -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/bin/sql-client: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ${FLINK_HOME}/bin/sql-client.sh embedded -d ${FLINK_HOME}/conf/sql-client-conf.yaml -l ${SQL_CLIENT_HOME}/lib -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/conf/flink-conf.yaml: -------------------------------------------------------------------------------- 1 | jobmanager.rpc.address: jobmanager 2 | -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/conf/sql-client-conf.yaml: -------------------------------------------------------------------------------- 1 | execution: 2 | planner: blink # using the Blink planner 3 | type: streaming # 'batch' or 'streaming' execution 4 | result-mode: table # 'changelog' or 'table' presentation of results 5 | parallelism: 1 # parallelism of the program 6 | max-parallelism: 128 # maximum parallelism 7 | min-idle-state-retention: 0 # minimum idle state retention in ms 8 | max-idle-state-retention: 0 # maximum idle state retention in ms 9 | 10 | #============================================================================== 11 | # Deployment properties 12 | #============================================================================== 13 | 14 | # Deployment properties allow for describing the cluster to which table 15 | # programs are submitted to. 16 | 17 | deployment: 18 | type: standalone # only the 'standalone' deployment is supported 19 | response-timeout: 5000 # general cluster communication timeout in ms 20 | gateway-address: "" # (optional) address from cluster to gateway 21 | gateway-port: 0 -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tail -f /dev/null -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/lib/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/sql-client/lib/.DS_Store -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/lib/flink-shaded-hadoop-2-uber-2.7.5-10.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/sql-client/lib/flink-shaded-hadoop-2-uber-2.7.5-10.0.jar -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/lib/flink-sql-connector-mysql-cdc-2.1-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/sql-client/lib/flink-sql-connector-mysql-cdc-2.1-SNAPSHOT.jar -------------------------------------------------------------------------------- /flink-cdc-iceberg-demo/sql-client/lib/iceberg-flink-1.13-runtime-0.13.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luoyuxia/flink-cdc-tutorial/fdfc739a8833889cfeff55740529931403438831/flink-cdc-iceberg-demo/sql-client/lib/iceberg-flink-1.13-runtime-0.13.0-SNAPSHOT.jar --------------------------------------------------------------------------------