├── .gitignore ├── README.md ├── bookkeeper-example ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── matt │ └── test │ └── bookkeeper │ └── ledger │ └── LeagerTest.java ├── calcite-example ├── pom.xml └── src │ └── main │ ├── codegen │ └── templates │ │ ├── adder.jj │ │ └── calculator.jj │ ├── java │ └── com │ │ └── matt │ │ └── test │ │ └── calcite │ │ ├── CalciteUtils.java │ │ └── sql │ │ ├── SqlHepTest.java │ │ └── SqlVolcanoTest.java │ └── resources │ └── log4j.properties ├── flink-example ├── .idea │ ├── compiler.xml │ ├── copyright │ │ └── profiles_settings.xml │ ├── libraries │ │ ├── Maven__com_esotericsoftware_kryo_kryo_2_24_0.xml │ │ ├── Maven__com_esotericsoftware_minlog_minlog_1_2.xml │ │ ├── Maven__com_fasterxml_jackson_core_jackson_annotations_2_7_9.xml │ │ ├── Maven__com_fasterxml_jackson_core_jackson_core_2_7_9.xml │ │ ├── Maven__com_fasterxml_jackson_core_jackson_databind_2_7_9.xml │ │ ├── Maven__com_github_scopt_scopt_2_11_3_5_0.xml │ │ ├── Maven__com_google_code_findbugs_jsr305_1_3_9.xml │ │ ├── Maven__com_twitter_chill_2_11_0_7_4.xml │ │ ├── Maven__com_twitter_chill_java_0_7_4.xml │ │ ├── Maven__com_typesafe_akka_akka_actor_2_11_2_4_20.xml │ │ ├── Maven__com_typesafe_akka_akka_protobuf_2_11_2_4_20.xml │ │ ├── Maven__com_typesafe_akka_akka_slf4j_2_11_2_4_20.xml │ │ ├── Maven__com_typesafe_akka_akka_stream_2_11_2_4_20.xml │ │ ├── Maven__com_typesafe_config_1_3_0.xml │ │ ├── Maven__com_typesafe_ssl_config_core_2_11_0_2_1.xml │ │ ├── Maven__commons_cli_commons_cli_1_3_1.xml │ │ ├── Maven__commons_collections_commons_collections_3_2_2.xml │ │ ├── Maven__commons_io_commons_io_2_4.xml │ │ ├── Maven__net_jpountz_lz4_lz4_1_3_0.xml │ │ ├── Maven__org_apache_commons_commons_compress_1_4_1.xml │ │ ├── Maven__org_apache_commons_commons_lang3_3_3_2.xml │ │ ├── Maven__org_apache_commons_commons_math3_3_5.xml │ │ ├── Maven__org_apache_flink_flink_annotations_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_clients_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_connector_kafka_0_10_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_connector_kafka_0_9_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_connector_kafka_base_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_connector_twitter_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_connector_wikiedits_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_core_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_hadoop_fs_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_java_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_metrics_core_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_optimizer_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_queryable_state_client_java_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_runtime_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_flink_shaded_asm_5_0_4_2_0.xml │ │ ├── Maven__org_apache_flink_flink_shaded_guava_18_0_2_0.xml │ │ ├── Maven__org_apache_flink_flink_shaded_jackson_2_7_9_3_0.xml │ │ ├── Maven__org_apache_flink_flink_shaded_netty_4_0_27_Final_2_0.xml │ │ ├── Maven__org_apache_flink_flink_streaming_java_2_11_1_5_0.xml │ │ ├── Maven__org_apache_flink_force_shading_1_5_0.xml │ │ ├── Maven__org_apache_kafka_kafka_clients_0_10_2_1.xml │ │ ├── Maven__org_clapper_grizzled_slf4j_2_11_1_0_2.xml │ │ ├── Maven__org_javassist_javassist_3_18_2_GA.xml │ │ ├── Maven__org_objenesis_objenesis_2_1.xml │ │ ├── Maven__org_reactivestreams_reactive_streams_1_0_0.xml │ │ ├── Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml │ │ ├── Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_4.xml │ │ ├── Maven__org_scala_lang_scala_library_2_11_12.xml │ │ ├── Maven__org_schwering_irclib_1_10.xml │ │ ├── Maven__org_slf4j_slf4j_api_1_7_7.xml │ │ ├── Maven__org_tukaani_xz_1_0.xml │ │ └── Maven__org_xerial_snappy_snappy_java_1_1_4.xml │ ├── misc.xml │ ├── modules.xml │ ├── thriftCompiler.xml │ ├── vcs.xml │ └── workspace.xml ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── matt │ │ └── test │ │ ├── kafka │ │ └── KafkaEvent.java │ │ ├── table │ │ ├── StreamSQLExample.java │ │ ├── WordCountSQL.java │ │ └── WordCountTable.java │ │ └── worcount │ │ └── SocketTextStreamWordCount.java │ └── resources │ └── log4j.properties ├── kafka-example ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── matt │ │ └── test │ │ └── kafka │ │ ├── producer │ │ ├── ProducerIdempotenceExample.java │ │ ├── ProducerJsonExample.java │ │ ├── ProducerRecoverTest.java │ │ └── ProducerTransactionExample.java │ │ └── zookeeper │ │ └── ZooKeeperWatcherTest.java │ └── resources │ └── log4j.properties ├── leetcode ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── matt │ └── test │ └── leetcode │ └── string │ └── RotateString.java └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/intellij,eclipse,maven 2 | 3 | ### Intellij ### 4 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 5 | 6 | *.iml 7 | .idea/ 8 | 9 | ### Maven ### 10 | target/ 11 | pom.xml.tag 12 | pom.xml.releaseBackup 13 | pom.xml.versionsBackup 14 | pom.xml.next 15 | release.properties 16 | dependency-reduced-pom.xml 17 | buildNumber.properties 18 | .mvn/timing.properties 19 | 20 | # mac 21 | .DS_Store 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # program-example 2 | 3 | 1. bookkeeper-example：BookKeeper 的一些示例； 4 | 2. calcite-example：Calcite 的几个示例； 5 | 3. flink-example：Flink 的一些测试示例； 6 | 4. kafka-example：Kafka 的一些测试示例； 7 | 5. leetcode：对照极客时间课程《数据结构与算法之美》做的相应练习，计划每个月一个专题，每周刷一道题，弥补自己算法及数据结构能力的不足； -------------------------------------------------------------------------------- /bookkeeper-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.matt.test 8 | bookkeeper-example 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 4.8.0 13 | 14 | 15 | 16 | 17 | 18 | org.apache.bookkeeper 19 | bookkeeper-server 20 | ${bookkeeper.version} 21 | 22 | 23 | 24 | org.slf4j 25 | slf4j-api 26 | 1.7.21 27 | 28 | 29 | org.slf4j 30 | slf4j-log4j12 31 | 1.7.21 32 | 33 | 34 | 35 | 36 | 37 | org.apache.maven.plugins 38 | maven-shade-plugin 39 | 2.4 40 | 41 | 42 | package 43 | 44 | shade 45 | 46 | 47 | ${project.artifactId}-${project.version}-with-dependencies 48 | 49 | 50 | 51 | 52 | 53 | org.apache.maven.plugins 54 | maven-compiler-plugin 55 | 56 | 1.8 57 | 1.8 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /bookkeeper-example/src/main/java/com/matt/test/bookkeeper/ledger/LeagerTest.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.bookkeeper.ledger; 2 | 3 | import java.io.IOException; 4 | import java.util.Enumeration; 5 | import org.apache.bookkeeper.client.AsyncCallback; 6 | import org.apache.bookkeeper.client.BKException; 7 | import org.apache.bookkeeper.client.BookKeeper; 8 | import org.apache.bookkeeper.client.LedgerEntry; 9 | import org.apache.bookkeeper.client.LedgerHandle; 10 | import org.apache.bookkeeper.conf.ClientConfiguration; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | public class LeagerTest { 15 | 16 | private BookKeeper bkClient; 17 | 18 | private static final Logger logger = LoggerFactory.getLogger(LeagerTest.class); 19 | 20 | 21 | /** 22 | * init the BookKeeper client 23 | */ 24 | public LeagerTest(String zkAddr) { 25 | try { 26 | // /* first method */ 27 | // String connectionString = zkAddr; // For a single-node, local ZooKeeper cluster 28 | // BookKeeper bkClient = new BookKeeper(connectionString); 29 | /* second method*/ 30 | ClientConfiguration config = new ClientConfiguration(); 31 | config.setZkServers(zkAddr); 32 | config.setAddEntryTimeout(2000); 33 | BookKeeper bkClient = new BookKeeper(config); 34 | this.bkClient = bkClient; 35 | logger.info("BookKeeper client init success."); 36 | } catch (InterruptedException | IOException | BKException e) { 37 | e.printStackTrace(); 38 | throw new RuntimeException( 39 | "There is an exception throw while creating the BookKeeper client."); 40 | } 41 | } 42 | 43 | /** 44 | * create the ledger 45 | * 46 | * @param pw password 47 | * @return LedgerHandle 48 | */ 49 | public LedgerHandle createLedgerSync(String pw) { 50 | byte[] password = pw.getBytes(); 51 | try { 52 | LedgerHandle handle = bkClient.createLedger(BookKeeper.DigestType.MAC, password); 53 | return handle; 54 | } catch (BKException e) { 55 | e.printStackTrace(); 56 | } catch (InterruptedException e) { 57 | e.printStackTrace(); 58 | } 59 | return null; 60 | } 61 | 62 | /** 63 | * create the ledger 64 | * 65 | * @param pw password 66 | */ 67 | public void createLedgerAsync(String pw) { 68 | 69 | class LedgerCreationCallback implements AsyncCallback.CreateCallback { 70 | 71 | public void createComplete(int returnCode, LedgerHandle handle, Object ctx) { 72 | System.out.println("Ledger successfully created"); 73 | logger.info("Ledger successfully created async."); 74 | } 75 | } 76 | 77 | bkClient.asyncCreateLedger( 78 | 3, 79 | 2, 80 | BookKeeper.DigestType.MAC, 81 | pw.getBytes(), 82 | new LedgerCreationCallback(), 83 | "some context" 84 | ); 85 | } 86 | 87 | /** 88 | * add the entry to the ledger 89 | * 90 | * @param ledgerHandle the ledger 91 | * @param msg msg 92 | * @return entryId, if occur exception, return -1 93 | */ 94 | public long addEntry(LedgerHandle ledgerHandle, String msg) { 95 | try { 96 | return ledgerHandle.addEntry(msg.getBytes()); 97 | } catch (InterruptedException e) { 98 | e.printStackTrace(); 99 | } catch (BKException e) { 100 | e.printStackTrace(); 101 | } 102 | return -1; 103 | } 104 | 105 | /** 106 | * read entry from startId to endId 107 | * 108 | * @param ledgerHandle the ledger 109 | * @param startId start entry id 110 | * @param endId end entry id 111 | * @return the entries, if occur exception, return null 112 | */ 113 | public Enumeration readEntry(LedgerHandle ledgerHandle, int startId, int endId) { 114 | try { 115 | return ledgerHandle.readEntries(startId, endId); 116 | } catch (InterruptedException e) { 117 | e.printStackTrace(); 118 | } catch (BKException e) { 119 | e.printStackTrace(); 120 | } 121 | return null; 122 | } 123 | 124 | /** 125 | * read entry from 0 to the LAC 126 | * 127 | * @param ledgerHandle the ledger 128 | * @return the entries, if occur exception, return null 129 | */ 130 | public Enumeration readEntry(LedgerHandle ledgerHandle) { 131 | try { 132 | return ledgerHandle.readEntries(0, ledgerHandle.getLastAddConfirmed()); 133 | } catch (InterruptedException e) { 134 | e.printStackTrace(); 135 | } catch (BKException e) { 136 | e.printStackTrace(); 137 | } 138 | return null; 139 | } 140 | 141 | /** 142 | * read entry after the LastAddConfirmed range 143 | * 144 | * @param ledgerHandle the handle 145 | * @param lastEntryIdExpectedToRead the last entry id 146 | * @return the entries, if occur exception, return null 147 | */ 148 | public Enumeration readEntry(LedgerHandle ledgerHandle, 149 | long lastEntryIdExpectedToRead) { 150 | try { 151 | return ledgerHandle.readUnconfirmedEntries(0, lastEntryIdExpectedToRead); 152 | } catch (InterruptedException e) { 153 | e.printStackTrace(); 154 | } catch (BKException e) { 155 | e.printStackTrace(); 156 | } 157 | return null; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /calcite-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.matt.test 8 | calcite-example 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 1.18.0 15 | 1.7.25 16 | 4.13.1 17 | 6.0 18 | 19.0 19 | 20 | 21 | 22 | 23 | org.apache.calcite 24 | calcite-core 25 | ${calcite-core.version} 26 | 27 | 28 | 29 | junit 30 | junit 31 | ${junit.version} 32 | compile 33 | 34 | 35 | org.slf4j 36 | slf4j-api 37 | ${slf4j.version} 38 | 39 | 40 | org.slf4j 41 | slf4j-log4j12 42 | ${slf4j.version} 43 | 44 | 45 | com.google.guava 46 | guava 47 | ${guava.version} 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-compiler-plugin 56 | 3.2 57 | 58 | 1.8 59 | 1.8 60 | 61 | 62 | 63 | org.apache.maven.plugins 64 | maven-shade-plugin 65 | 3.2.0 66 | 67 | 68 | package 69 | 70 | shade 71 | 72 | 73 | 74 | 75 | *:* 76 | 77 | META-INF/*.SF 78 | META-INF/*.DSA 79 | META-INF/*.RSA 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /calcite-example/src/main/codegen/templates/adder.jj: -------------------------------------------------------------------------------- 1 | options { 2 | STATIC = false; 3 | } 4 | 5 | PARSER_BEGIN(Adder) 6 | 7 | import java.io.*; 8 | 9 | public class Adder { 10 | public static void main(String[] args){ 11 | for (String arg : args) { 12 | try{ 13 | System.out.println(evaluate(arg)); 14 | } catch (ParseException e) { 15 | System.err.println(e.getMessage()); 16 | } 17 | } 18 | } 19 | 20 | public static long evaluate(String src) throws ParseException { 21 | Reader reader = new StringReader(src); 22 | return new Adder(reader).expr(); 23 | } 24 | } 25 | PARSER_END(Adder) 26 | 27 | SKIP : { 28 | <[" ", "\t", "\r", "\n"]> 29 | } 30 | 31 | TOKEN : { 32 | 33 | } 34 | 35 | long expr(): 36 | { 37 | Token x, y; 38 | } 39 | 40 | { 41 | x= "+" y= 42 | { 43 | return Long.parseLong(x.image) + Long.parseLong(y.image); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /calcite-example/src/main/codegen/templates/calculator.jj: -------------------------------------------------------------------------------- 1 | /** 2 | * Author: matt 3 | */ 4 | 5 | options { 6 | STATIC = false; 7 | } 8 | 9 | PARSER_BEGIN(Calculator) 10 | 11 | import java.io.*; 12 | 13 | public class Calculator { 14 | public static void main(String[] args) throws ParseException, TokenMgrError, NumberFormatException { 15 | Calculator calculator = new Calculator(System.in); 16 | calculator.parse(System.out); 17 | } 18 | 19 | double previousValue = 0.0; 20 | } 21 | 22 | PARSER_END(Calculator) 23 | 24 | SKIP : { 25 | " " 26 | } 27 | 28 | // define EOF 29 | TOKEN : { 30 | < EOL : "\n" | "\r" | "\r\n"> 31 | } 32 | 33 | TOKEN : { 34 | < PLUS : "+" > 35 | | 36 | < TIMES : "*" > 37 | | 38 | < DIVIDE : "/" > 39 | | 40 | < MINUS : "-" > 41 | | 42 | < OPEN_PAR : "(" > 43 | | 44 | < CLOSE_PAR : ")" > 45 | } 46 | 47 | TOKEN : { 48 | < NUMBER : (["0"-"9"])+ | (["0"-"9"])+ "." (["0"-"9"])+ | (["0"-"9"])+ "." | "." (["0"-"9"])+ > // allow decimal points in number 49 | } 50 | 51 | void parse(PrintStream printStream) throws NumberFormatException : 52 | {} 53 | { 54 | ( 55 | previousValue = expression() 56 | 57 | { printStream.println(previousValue); } 58 | )* 59 | 60 | } 61 | 62 | double expression() throws NumberFormatException : 63 | { 64 | double i; 65 | double value; 66 | } 67 | { 68 | value = term() 69 | ( 70 | 71 | i = term() 72 | { value += i; } 73 | | 74 | 75 | i = term() 76 | { value -= i; } 77 | )* 78 | { return value ; } 79 | 80 | } 81 | 82 | double term() throws NumberFormatException : 83 | { 84 | double i; 85 | double value; 86 | } 87 | { 88 | value = primary() 89 | ( 90 | 91 | i = primary() 92 | { value *= i; } 93 | | 94 | 95 | i = primary() 96 | { value /= i; } 97 | )* 98 | { return value; } 99 | } 100 | 101 | double primary() throws NumberFormatException : 102 | { 103 | Token t; 104 | double d; 105 | } 106 | { 107 | t = 108 | { return Double.parseDouble( t.image ); } 109 | | 110 | d=expression() 111 | { return d; } 112 | | 113 | d=primary() 114 | { return -d; } 115 | } 116 | 117 | -------------------------------------------------------------------------------- /calcite-example/src/main/java/com/matt/test/calcite/CalciteUtils.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.calcite; 2 | 3 | import org.apache.calcite.config.CalciteConnectionConfig; 4 | import org.apache.calcite.plan.Context; 5 | import org.apache.calcite.plan.RelOptTable; 6 | import org.apache.calcite.rel.RelRoot; 7 | import org.apache.calcite.rel.type.RelDataType; 8 | import org.apache.calcite.rel.type.RelDataTypeFactory; 9 | import org.apache.calcite.rel.type.RelDataTypeSystemImpl; 10 | import org.apache.calcite.rex.RexBuilder; 11 | import org.apache.calcite.schema.SchemaPlus; 12 | import org.apache.calcite.schema.impl.AbstractTable; 13 | import org.apache.calcite.sql.type.BasicSqlType; 14 | import org.apache.calcite.sql.type.SqlTypeName; 15 | import org.apache.calcite.sql.validate.SqlConformance; 16 | import org.apache.calcite.sql.validate.SqlConformanceEnum; 17 | import org.apache.calcite.tools.FrameworkConfig; 18 | import org.apache.calcite.tools.Frameworks; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * register the users and jobs table 24 | * 25 | * @author matt 26 | * @date 2019-03-19 19:58 27 | */ 28 | public class CalciteUtils { 29 | 30 | public static SchemaPlus registerRootSchema() { 31 | SchemaPlus rootSchema = Frameworks.createRootSchema(true); 32 | rootSchema.add("USERS", new AbstractTable() { //note: add a table 33 | @Override 34 | public RelDataType getRowType(final RelDataTypeFactory typeFactory) { 35 | RelDataTypeFactory.Builder builder = typeFactory.builder(); 36 | 37 | builder.add("ID", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.INTEGER)); 38 | builder.add("NAME", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.CHAR)); 39 | builder.add("AGE", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.INTEGER)); 40 | return builder.build(); 41 | } 42 | }); 43 | 44 | rootSchema.add("JOBS", new AbstractTable() { 45 | @Override 46 | public RelDataType getRowType(final RelDataTypeFactory typeFactory) { 47 | RelDataTypeFactory.Builder builder = typeFactory.builder(); 48 | 49 | builder.add("ID", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.INTEGER)); 50 | builder.add("NAME", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.CHAR)); 51 | builder.add("COMPANY", new BasicSqlType(new RelDataTypeSystemImpl() {}, SqlTypeName.CHAR)); 52 | return builder.build(); 53 | } 54 | }); 55 | return rootSchema; 56 | } 57 | 58 | public static SqlConformance conformance(FrameworkConfig config) { 59 | final Context context = config.getContext(); 60 | if (context != null) { 61 | final CalciteConnectionConfig connectionConfig = 62 | context.unwrap(CalciteConnectionConfig.class); 63 | if (connectionConfig != null) { 64 | return connectionConfig.conformance(); 65 | } 66 | } 67 | return SqlConformanceEnum.DEFAULT; 68 | } 69 | 70 | public static RexBuilder createRexBuilder(RelDataTypeFactory typeFactory) { 71 | return new RexBuilder(typeFactory); 72 | } 73 | 74 | public static class ViewExpanderImpl implements RelOptTable.ViewExpander { 75 | public ViewExpanderImpl() { 76 | } 77 | 78 | @Override 79 | public RelRoot expandView(RelDataType rowType, String queryString, List schemaPath, 80 | List viewPath) { 81 | return null; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /calcite-example/src/main/java/com/matt/test/calcite/sql/SqlHepTest.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.calcite.sql; 2 | 3 | import com.matt.test.calcite.CalciteUtils; 4 | import org.apache.calcite.config.CalciteConnectionConfigImpl; 5 | import org.apache.calcite.jdbc.CalciteSchema; 6 | import org.apache.calcite.plan.ConventionTraitDef; 7 | import org.apache.calcite.plan.RelOptCluster; 8 | import org.apache.calcite.plan.RelOptUtil; 9 | import org.apache.calcite.plan.hep.HepPlanner; 10 | import org.apache.calcite.plan.hep.HepProgramBuilder; 11 | import org.apache.calcite.prepare.CalciteCatalogReader; 12 | import org.apache.calcite.rel.RelDistributionTraitDef; 13 | import org.apache.calcite.rel.RelNode; 14 | import org.apache.calcite.rel.RelRoot; 15 | import org.apache.calcite.rel.rules.FilterJoinRule; 16 | import org.apache.calcite.rel.rules.PruneEmptyRules; 17 | import org.apache.calcite.rel.rules.ReduceExpressionsRule; 18 | import org.apache.calcite.rel.type.RelDataTypeSystem; 19 | import org.apache.calcite.rex.RexBuilder; 20 | import org.apache.calcite.schema.SchemaPlus; 21 | import org.apache.calcite.sql.SqlNode; 22 | import org.apache.calcite.sql.fun.SqlStdOperatorTable; 23 | import org.apache.calcite.sql.parser.SqlParser; 24 | import org.apache.calcite.sql.type.SqlTypeFactoryImpl; 25 | import org.apache.calcite.sql.validate.SqlValidator; 26 | import org.apache.calcite.sql.validate.SqlValidatorUtil; 27 | import org.apache.calcite.sql2rel.RelDecorrelator; 28 | import org.apache.calcite.sql2rel.SqlToRelConverter; 29 | import org.apache.calcite.tools.FrameworkConfig; 30 | import org.apache.calcite.tools.Frameworks; 31 | import org.apache.calcite.tools.RelBuilder; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | import java.util.Properties; 36 | 37 | /** 38 | * Calcite example use HepPlanner 39 | * 40 | * @author matt 41 | * @date 2019-03-19 19:58 42 | */ 43 | public class SqlHepTest { 44 | private static final Logger LOGGER = LoggerFactory.getLogger(SqlHepTest.class); 45 | 46 | public static void main(String[] args) { 47 | SchemaPlus rootSchema = CalciteUtils.registerRootSchema(); 48 | 49 | final FrameworkConfig fromworkConfig = Frameworks.newConfigBuilder() 50 | .parserConfig(SqlParser.Config.DEFAULT) 51 | .defaultSchema(rootSchema) 52 | .traitDefs(ConventionTraitDef.INSTANCE, RelDistributionTraitDef.INSTANCE) 53 | .build(); 54 | 55 | String sql 56 | = "select u.id as user_id, u.name as user_name, j.company as user_company, u.age as user_age from users u" 57 | + " join jobs j on u.id=j.id where u.age > 30 and j.id>10 order by user_id"; 58 | 59 | // use HepPlanner 60 | HepProgramBuilder builder = new HepProgramBuilder(); 61 | builder.addRuleInstance(FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN); 62 | builder.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE); 63 | builder.addRuleInstance(PruneEmptyRules.PROJECT_INSTANCE); 64 | HepPlanner planner = new HepPlanner(builder.build()); 65 | 66 | try { 67 | SqlTypeFactoryImpl factory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); 68 | // sql parser 69 | SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT); 70 | SqlNode parsed = parser.parseStmt(); 71 | LOGGER.info("The SqlNode after parsed is:\n{}", parsed.toString()); 72 | 73 | CalciteCatalogReader calciteCatalogReader = new CalciteCatalogReader( 74 | CalciteSchema.from(rootSchema), 75 | CalciteSchema.from(rootSchema).path(null), 76 | factory, 77 | new CalciteConnectionConfigImpl(new Properties())); 78 | 79 | // sql validate 80 | SqlValidator validator = SqlValidatorUtil.newValidator(SqlStdOperatorTable.instance(), calciteCatalogReader, 81 | factory, CalciteUtils.conformance(fromworkConfig)); 82 | SqlNode validated = validator.validate(parsed); 83 | LOGGER.info("The SqlNode after validated is:\n{}", validated.toString()); 84 | 85 | final RexBuilder rexBuilder = CalciteUtils.createRexBuilder(factory); 86 | final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); 87 | 88 | // init SqlToRelConverter config 89 | final SqlToRelConverter.Config config = SqlToRelConverter.configBuilder() 90 | .withConfig(fromworkConfig.getSqlToRelConverterConfig()) 91 | .withTrimUnusedFields(false) 92 | .withConvertTableAccess(false) 93 | .build(); 94 | // SqlNode toRelNode 95 | final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(new CalciteUtils.ViewExpanderImpl(), 96 | validator, calciteCatalogReader, cluster, fromworkConfig.getConvertletTable(), config); 97 | RelRoot root = sqlToRelConverter.convertQuery(validated, false, true); 98 | 99 | root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); 100 | final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); 101 | root = root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); 102 | RelNode relNode = root.rel; 103 | LOGGER.info("The relational expression string before optimized is:\n{}", RelOptUtil.toString(relNode)); 104 | 105 | planner.setRoot(relNode); 106 | relNode = planner.findBestExp(); 107 | System.out.println("-----------------------------------------------------------"); 108 | System.out.println("The Best relational expression string:"); 109 | System.out.println(RelOptUtil.toString(relNode)); 110 | System.out.println("-----------------------------------------------------------"); 111 | 112 | } catch (Exception e) { 113 | e.printStackTrace(); 114 | } 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /calcite-example/src/main/java/com/matt/test/calcite/sql/SqlVolcanoTest.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.calcite.sql; 2 | 3 | import com.matt.test.calcite.CalciteUtils; 4 | import org.apache.calcite.adapter.enumerable.EnumerableConvention; 5 | import org.apache.calcite.adapter.enumerable.EnumerableRules; 6 | import org.apache.calcite.config.CalciteConnectionConfigImpl; 7 | import org.apache.calcite.jdbc.CalciteSchema; 8 | import org.apache.calcite.plan.ConventionTraitDef; 9 | import org.apache.calcite.plan.RelOptCluster; 10 | import org.apache.calcite.plan.RelOptUtil; 11 | import org.apache.calcite.plan.RelTraitSet; 12 | import org.apache.calcite.plan.volcano.VolcanoPlanner; 13 | import org.apache.calcite.prepare.CalciteCatalogReader; 14 | import org.apache.calcite.rel.RelDistributionTraitDef; 15 | import org.apache.calcite.rel.RelNode; 16 | import org.apache.calcite.rel.RelRoot; 17 | import org.apache.calcite.rel.rules.FilterJoinRule; 18 | import org.apache.calcite.rel.rules.PruneEmptyRules; 19 | import org.apache.calcite.rel.rules.ReduceExpressionsRule; 20 | import org.apache.calcite.rel.type.RelDataTypeSystem; 21 | import org.apache.calcite.rex.RexBuilder; 22 | import org.apache.calcite.schema.SchemaPlus; 23 | import org.apache.calcite.sql.SqlNode; 24 | import org.apache.calcite.sql.fun.SqlStdOperatorTable; 25 | import org.apache.calcite.sql.parser.SqlParser; 26 | import org.apache.calcite.sql.type.SqlTypeFactoryImpl; 27 | import org.apache.calcite.sql.validate.SqlValidator; 28 | import org.apache.calcite.sql.validate.SqlValidatorUtil; 29 | import org.apache.calcite.sql2rel.RelDecorrelator; 30 | import org.apache.calcite.sql2rel.SqlToRelConverter; 31 | import org.apache.calcite.tools.FrameworkConfig; 32 | import org.apache.calcite.tools.Frameworks; 33 | import org.apache.calcite.tools.RelBuilder; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | import java.util.Properties; 38 | 39 | /** 40 | * calcite example use VolcanoPlanner 41 | * 42 | * @author matt 43 | * @date 2019-03-18 21:33 44 | */ 45 | public class SqlVolcanoTest { 46 | private static final Logger LOGGER = LoggerFactory.getLogger(SqlVolcanoTest.class); 47 | 48 | public static void main(String[] args) { 49 | SchemaPlus rootSchema = CalciteUtils.registerRootSchema(); 50 | 51 | final FrameworkConfig fromworkConfig = Frameworks.newConfigBuilder() 52 | .parserConfig(SqlParser.Config.DEFAULT) 53 | .defaultSchema(rootSchema) 54 | .traitDefs(ConventionTraitDef.INSTANCE, RelDistributionTraitDef.INSTANCE) 55 | .build(); 56 | 57 | String sql 58 | = "select u.id as user_id, u.name as user_name, j.company as user_company, u.age as user_age from users u" 59 | + " join jobs j on u.id=j.id where u.age > 30 and j.id>10 order by user_id"; 60 | 61 | // use HepPlanner 62 | VolcanoPlanner planner = new VolcanoPlanner(); 63 | planner.addRelTraitDef(ConventionTraitDef.INSTANCE); 64 | planner.addRelTraitDef(RelDistributionTraitDef.INSTANCE); 65 | // add rules 66 | planner.addRule(FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN); 67 | planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE); 68 | planner.addRule(PruneEmptyRules.PROJECT_INSTANCE); 69 | // add ConverterRule 70 | planner.addRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE); 71 | planner.addRule(EnumerableRules.ENUMERABLE_SORT_RULE); 72 | planner.addRule(EnumerableRules.ENUMERABLE_VALUES_RULE); 73 | planner.addRule(EnumerableRules.ENUMERABLE_PROJECT_RULE); 74 | planner.addRule(EnumerableRules.ENUMERABLE_FILTER_RULE); 75 | 76 | try { 77 | SqlTypeFactoryImpl factory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); 78 | // sql parser 79 | SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT); 80 | SqlNode parsed = parser.parseStmt(); 81 | LOGGER.info("The SqlNode after parsed is:\n{}", parsed.toString()); 82 | 83 | CalciteCatalogReader calciteCatalogReader = new CalciteCatalogReader( 84 | CalciteSchema.from(rootSchema), 85 | CalciteSchema.from(rootSchema).path(null), 86 | factory, 87 | new CalciteConnectionConfigImpl(new Properties())); 88 | 89 | // sql validate 90 | SqlValidator validator = SqlValidatorUtil.newValidator(SqlStdOperatorTable.instance(), calciteCatalogReader, 91 | factory, CalciteUtils.conformance(fromworkConfig)); 92 | SqlNode validated = validator.validate(parsed); 93 | LOGGER.info("The SqlNode after validated is:\n{}", validated.toString()); 94 | 95 | final RexBuilder rexBuilder = CalciteUtils.createRexBuilder(factory); 96 | final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); 97 | 98 | // init SqlToRelConverter config 99 | final SqlToRelConverter.Config config = SqlToRelConverter.configBuilder() 100 | .withConfig(fromworkConfig.getSqlToRelConverterConfig()) 101 | .withTrimUnusedFields(false) 102 | .withConvertTableAccess(false) 103 | .build(); 104 | // SqlNode toRelNode 105 | final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(new CalciteUtils.ViewExpanderImpl(), 106 | validator, calciteCatalogReader, cluster, fromworkConfig.getConvertletTable(), config); 107 | RelRoot root = sqlToRelConverter.convertQuery(validated, false, true); 108 | 109 | root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); 110 | final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); 111 | root = root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); 112 | RelNode relNode = root.rel; 113 | LOGGER.info("The relational expression string before optimized is:\n{}", RelOptUtil.toString(relNode)); 114 | 115 | RelTraitSet desiredTraits = 116 | relNode.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); 117 | relNode = planner.changeTraits(relNode, desiredTraits); 118 | 119 | planner.setRoot(relNode); 120 | relNode = planner.findBestExp(); 121 | System.out.println("-----------------------------------------------------------"); 122 | System.out.println("The Best relational expression string:"); 123 | System.out.println(RelOptUtil.toString(relNode)); 124 | System.out.println("-----------------------------------------------------------"); 125 | 126 | } catch (Exception e) { 127 | e.printStackTrace(); 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /calcite-example/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=TRACE, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.Threshold=TRACE 5 | log4j.appender.console.ImmediateFlush=true 6 | log4j.appender.console.Target=System.err 7 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.console.layout.ConversionPattern=[%-5p] %d(%r) --> [%t] %l: %m %x %n 9 | 10 | -------------------------------------------------------------------------------- /flink-example/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /flink-example/.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_esotericsoftware_kryo_kryo_2_24_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_esotericsoftware_minlog_minlog_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_7_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_7_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_7_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_github_scopt_scopt_2_11_3_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_twitter_chill_2_11_0_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_twitter_chill_java_0_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_akka_akka_protobuf_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_akka_akka_slf4j_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_akka_akka_stream_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__com_typesafe_ssl_config_core_2_11_0_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__commons_cli_commons_cli_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__commons_io_commons_io_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__net_jpountz_lz4_lz4_1_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_commons_commons_math3_3_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_annotations_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_clients_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_connector_kafka_0_10_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_connector_kafka_0_9_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_connector_kafka_base_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_connector_twitter_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_connector_wikiedits_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_core_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_hadoop_fs_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_java_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_metrics_core_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_optimizer_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_queryable_state_client_java_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_runtime_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_shaded_asm_5_0_4_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_shaded_guava_18_0_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_shaded_jackson_2_7_9_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_shaded_netty_4_0_27_Final_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_flink_streaming_java_2_11_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_flink_force_shading_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_apache_kafka_kafka_clients_0_10_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_clapper_grizzled_slf4j_2_11_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_javassist_javassist_3_18_2_GA.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_reactivestreams_reactive_streams_1_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_schwering_irclib_1_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_tukaani_xz_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /flink-example/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /flink-example/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /flink-example/.idea/thriftCompiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /flink-example/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /flink-example/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 102 | 103 | 104 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 197 | 198 | 199 | 201 | 202 | 214 | 217 | 218 | 219 | 231 | 232 | 233 | 234 | 235 | true 236 | DEFINITION_ORDER 237 | 238 | 239 | 244 | 249 | 250 | 251 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 291 | 292 | 293 | 294 | 297 | 298 | 301 | 302 | 303 | 304 | 307 | 308 | 311 | 312 | 315 | 316 | 317 | 318 | 321 | 322 | 325 | 326 | 329 | 330 | 333 | 334 | 335 | 336 | 339 | 340 | 343 | 344 | 347 | 348 | 351 | 352 | 355 | 356 | 357 | 358 | 361 | 362 | 365 | 366 | 369 | 370 | 373 | 374 | 377 | 378 | 381 | 382 | 385 | 386 | 387 | 388 | 391 | 392 | 395 | 396 | 399 | 400 | 403 | 404 | 407 | 408 | 411 | 412 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 441 | 442 | 443 | 460 | 461 | 462 | 478 | 479 | 486 | 487 | 488 | 501 | 502 | 503 | 504 | 509 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 535 | 553 | 560 | 561 | 571 | 572 | 581 | 582 | 583 | 600 | 601 | 620 | 621 | 642 | 655 | 656 | 665 | 669 | 670 | 671 | 678 | 681 | 683 | 684 | 685 | 686 | 687 | 688 | 689 | 690 | 691 | 692 | 693 | 694 | 695 | 696 | 697 | 711 | 712 | 713 | 714 | 715 | 716 | 717 | 718 | 732 | 733 | 734 | 735 | 736 | 737 | 738 | 739 | 740 | 741 | 742 | 743 | 757 | 758 | 759 | 760 | 761 | 762 | 773 | 774 | 775 | 785 | 786 | 799 | 800 | 813 | 814 | 831 | 832 | 839 | 840 | 841 | 842 | 843 | 844 | 845 | 846 | 847 | 848 | 849 | 850 | 851 | 852 | 853 | 854 | 855 | 856 | 857 | 858 | 859 | 860 | 861 | 862 | 863 | 864 | 865 | 866 | 867 | 868 | 869 | 870 | 871 | 872 | 873 | 874 | 875 | 876 | 877 | 878 | 896 | 903 | 904 | 905 | 906 | 924 | 931 | 932 | 944 | 945 | 946 | 947 | 948 | 949 | 950 | 951 | 952 | 953 | 954 | 955 | 956 | 957 | 958 | project 959 | 960 | 961 | 962 | 963 | 964 | 965 | 966 | 967 | 968 | 969 | 970 | 971 | 972 | 973 | 974 | 975 | 976 | 977 | 978 | 979 | 980 | 999 | 1000 | 1019 | 1020 | 1041 | 1042 | 1064 | 1065 | 1089 | 1090 | 1091 | 1092 | 1093 | 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | 1101 | 1102 | 1103 | 1104 | 1105 | 1106 | 1109 | 1110 | 1111 | 1113 | 1114 | 1115 | 1116 | 1532144889210 1117 | 1124 | 1125 | 1126 | 1127 | 1129 | 1130 | 1131 | 1132 | 1133 | 1134 | 1135 | 1136 | 1137 | 1138 | 1139 | 1140 | 1141 | 1142 | 1143 | 1144 | 1145 | 1146 | 1147 | 1148 | 1149 | 1150 | 1151 | 1152 | 1153 | 1154 | 1155 | 1156 | 1157 | 1158 | 1159 | 1160 | 1161 | 1162 | 1163 | 1164 | 1165 | 1166 | 1167 | 1168 | 1169 | 1170 | 1171 | 1172 | 1173 | 1174 | 1175 | 1176 | 1177 | 1178 | 1179 | 1180 | 1181 | 1182 | 1183 | 1184 | 1185 | 1186 | 1187 | 1188 | 1189 | 1190 | 1192 | 1193 | 1194 | 1195 | 1196 | 1197 | 1200 | 1201 | 1202 | 1203 | 1204 | 1205 | 1206 | 1207 | 1208 | 1209 | 1210 | 1211 | 1212 | 1213 | 1214 | 1215 | 1216 | 1217 | 1218 | 1219 | 1220 | 1221 | 1222 | 1223 | 1224 | 1225 | 1226 | 1227 | 1228 | 1229 | 1230 | 1231 | 1232 | 1233 | 1234 | 1235 | 1236 | 1237 | 1238 | 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | 1245 | 1246 | 1247 | 1248 | 1249 | 1250 | 1251 | 1252 | 1253 | 1254 | 1255 | 1256 | 1257 | 1258 | 1259 | 1260 | 1261 | 1262 | 1263 | 1264 | 1265 | 1266 | 1267 | 1268 | 1269 | 1270 | 1271 | 1272 | 1273 | 1274 | 1275 | 1276 | 1277 | 1278 | 1279 | 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | 1286 | 1287 | 1288 | 1289 | 1290 | 1291 | 1292 | 1293 | 1294 | 1295 | 1296 | 1297 | 1298 | 1299 | 1300 | 1301 | 1302 | 1303 | 1304 | 1305 | 1306 | 1307 | 1308 | 1309 | 1310 | 1311 | 1312 | 1313 | 1314 | 1315 | 1316 | 1317 | 1318 | 1319 | 1320 | 1321 | 1322 | 1323 | 1324 | 1325 | 1326 | 1327 | 1332 | 1333 | 1334 | 1335 | 1336 | 1337 | No facets are configured 1338 | 1339 | 1344 | 1345 | 1346 | 1347 | 1348 | 1349 | scala-sdk-2.10.4 1350 | 1351 | 1356 | 1357 | 1358 | 1359 | 1360 | 1361 | 1.7 1362 | 1363 | 1368 | 1369 | 1370 | 1371 | 1372 | 1373 | flink-example 1374 | 1375 | 1380 | 1381 | 1382 | 1383 | 1384 | 1385 | 1.8 1386 | 1387 | 1392 | 1393 | 1394 | 1395 | 1396 | 1397 | Maven: org.apache.flink:flink-streaming-java_2.11:1.5.1 1398 | 1399 | 1404 | 1405 | 1406 | 1407 | 1408 | 1409 | -------------------------------------------------------------------------------- /flink-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.matt.test 8 | flink-example 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.11 13 | 1.7.2 14 | 1.7.25 15 | 16 | 17 | 18 | 19 | 20 | org.slf4j 21 | slf4j-api 22 | ${slf4j.version} 23 | 24 | 25 | org.slf4j 26 | slf4j-log4j12 27 | ${slf4j.version} 28 | 29 | 30 | 31 | org.apache.flink 32 | flink-core 33 | ${flink-version} 34 | 35 | 36 | org.apache.flink 37 | flink-java 38 | ${flink-version} 39 | 40 | 41 | org.apache.flink 42 | flink-clients_${scala-vesion} 43 | ${flink-version} 44 | 45 | 46 | org.apache.flink 47 | flink-streaming-java_${scala-vesion} 48 | ${flink-version} 49 | 50 | 51 | 52 | org.apache.flink 53 | flink-connector-twitter_${scala-vesion} 54 | ${flink-version} 55 | 56 | 57 | 58 | org.apache.flink 59 | flink-connector-kafka-0.10_${scala-vesion} 60 | ${flink-version} 61 | compile 62 | 63 | 64 | 65 | org.apache.flink 66 | flink-connector-wikiedits_${scala-vesion} 67 | ${flink-version} 68 | 69 | 70 | 71 | org.apache.flink 72 | flink-shaded-jackson 73 | 2.7.9-3.0 74 | compile 75 | 76 | 77 | 78 | 79 | org.apache.flink 80 | flink-table_${scala-vesion} 81 | ${flink-version} 82 | 83 | 84 | 85 | org.apache.flink 86 | flink-streaming-scala_${scala-vesion} 87 | ${flink-version} 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | org.apache.maven.plugins 96 | maven-shade-plugin 97 | 2.4 98 | 99 | 100 | package 101 | 102 | shade 103 | 104 | 105 | ${project.artifactId}-${project.version}-with-dependencies 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-compiler-plugin 113 | 114 | 1.7 115 | 1.7 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /flink-example/src/main/java/com/matt/test/kafka/KafkaEvent.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka; 2 | 3 | /** 4 | * Created by matt on 22/07/2018. 5 | */ 6 | public class KafkaEvent { 7 | private String word; 8 | private int frequency; 9 | private long timestamp; 10 | 11 | public KafkaEvent() { 12 | } 13 | 14 | public KafkaEvent(String word, int frequency, long timestamp) { 15 | this.word = word; 16 | this.frequency = frequency; 17 | this.timestamp = timestamp; 18 | } 19 | 20 | public String getWord() { 21 | return word; 22 | } 23 | 24 | public void setWord(String word) { 25 | this.word = word; 26 | } 27 | 28 | public int getFrequency() { 29 | return frequency; 30 | } 31 | 32 | public void setFrequency(int frequency) { 33 | this.frequency = frequency; 34 | } 35 | 36 | public long getTimestamp() { 37 | return timestamp; 38 | } 39 | 40 | public void setTimestamp(long timestamp) { 41 | this.timestamp = timestamp; 42 | } 43 | 44 | public static KafkaEvent fromString(String eventStr) { 45 | String[] split = eventStr.split(","); 46 | return new KafkaEvent(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2])); 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return word + "," + frequency + "," + timestamp; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /flink-example/src/main/java/com/matt/test/table/StreamSQLExample.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.table; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 5 | import org.apache.flink.table.api.Table; 6 | import org.apache.flink.table.api.TableEnvironment; 7 | import org.apache.flink.table.api.java.StreamTableEnvironment; 8 | 9 | import java.util.Arrays; 10 | 11 | /** 12 | * Simple example for demonstrating the use of SQL on a Stream Table in Java. 13 | * 14 | *

This example shows how to: 15 | * - Convert DataStreams to Tables - Register a Table under a name - Run a StreamSQL query on the registered Table 16 | */ 17 | public class StreamSQLExample { 18 | 19 | // ************************************************************************* 20 | // PROGRAM 21 | // ************************************************************************* 22 | 23 | public static void main(String[] args) throws Exception { 24 | 25 | // set up execution environment 26 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 27 | StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); 28 | 29 | DataStream orderA = env.fromCollection(Arrays.asList( 30 | new Order(1L, "beer", 3), 31 | new Order(1L, "diaper", 4), 32 | new Order(3L, "rubber", 2))); 33 | 34 | DataStream orderB = env.fromCollection(Arrays.asList( 35 | new Order(2L, "pen", 3), 36 | new Order(2L, "rubber", 3), 37 | new Order(4L, "beer", 1))); 38 | 39 | // convert DataStream to Table 40 | Table tableA = tEnv.fromDataStream(orderA, "user, product, amount"); 41 | // register DataStream as Table 42 | tEnv.registerDataStream("OrderB", orderB, "user, product, amount"); 43 | 44 | // union the two tables 45 | Table result = tEnv.sqlQuery("SELECT * FROM " + tableA + " WHERE amount > 2 UNION ALL " + 46 | "SELECT * FROM OrderB WHERE amount < 2"); 47 | 48 | tEnv.toAppendStream(result, Order.class).print(); 49 | 50 | env.execute(); 51 | } 52 | 53 | // ************************************************************************* 54 | // USER DATA TYPES 55 | // ************************************************************************* 56 | 57 | /** 58 | * Simple POJO. 59 | */ 60 | public static class Order { 61 | public Long user; 62 | public String product; 63 | public int amount; 64 | 65 | public Order() { 66 | } 67 | 68 | public Order(Long user, String product, int amount) { 69 | this.user = user; 70 | this.product = product; 71 | this.amount = amount; 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return "Order{" + 77 | "user=" + user + 78 | ", product='" + product + '\'' + 79 | ", amount=" + amount + 80 | '}'; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /flink-example/src/main/java/com/matt/test/table/WordCountSQL.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.table; 2 | 3 | import org.apache.flink.api.java.DataSet; 4 | import org.apache.flink.api.java.ExecutionEnvironment; 5 | import org.apache.flink.table.api.Table; 6 | import org.apache.flink.table.api.TableEnvironment; 7 | import org.apache.flink.table.api.java.BatchTableEnvironment; 8 | 9 | /** 10 | * Simple example that shows how the Batch SQL API is used in Java. 11 | * 12 | *

This example shows how to: 13 | * - Convert DataSets to Tables - Register a Table under a name - Run a SQL query on the registered Table 14 | */ 15 | public class WordCountSQL { 16 | 17 | // ************************************************************************* 18 | // PROGRAM 19 | // ************************************************************************* 20 | 21 | public static void main(String[] args) throws Exception { 22 | 23 | // set up execution environment 24 | ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); 25 | BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); 26 | 27 | DataSet input = env.fromElements( 28 | new WC("Hello", 1), 29 | new WC("Ciao", 1), 30 | new WC("Hello", 1)); 31 | 32 | // register the DataSet as table "WordCount" 33 | tEnv.registerDataSet("WordCount", input, "word, frequency"); 34 | 35 | // run a SQL query on the Table and retrieve the result as a new Table 36 | Table table = tEnv.sqlQuery( 37 | "SELECT word, SUM(frequency) as frequency FROM WordCount GROUP BY word"); 38 | 39 | DataSet result = tEnv.toDataSet(table, WC.class); 40 | 41 | result.print(); 42 | } 43 | 44 | // ************************************************************************* 45 | // USER DATA TYPES 46 | // ************************************************************************* 47 | 48 | /** 49 | * Simple POJO containing a word and its respective count. 50 | */ 51 | public static class WC { 52 | public String word; 53 | public long frequency; 54 | 55 | // public constructor to make it a Flink POJO 56 | public WC() {} 57 | 58 | public WC(String word, long frequency) { 59 | this.word = word; 60 | this.frequency = frequency; 61 | } 62 | 63 | @Override 64 | public String toString() { 65 | return "WC " + word + " " + frequency; 66 | } 67 | } 68 | } 69 | 70 | -------------------------------------------------------------------------------- /flink-example/src/main/java/com/matt/test/table/WordCountTable.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.table; 2 | 3 | import org.apache.flink.api.java.DataSet; 4 | import org.apache.flink.api.java.ExecutionEnvironment; 5 | import org.apache.flink.table.api.Table; 6 | import org.apache.flink.table.api.TableEnvironment; 7 | import org.apache.flink.table.api.java.BatchTableEnvironment; 8 | 9 | /** 10 | * Simple example for demonstrating the use of the Table API for a Word Count in Java. 11 | * 12 | *

This example shows how to: 13 | * - Convert DataSets to Tables - Apply group, aggregate, select, and filter operations 14 | */ 15 | public class WordCountTable { 16 | 17 | // ************************************************************************* 18 | // PROGRAM 19 | // ************************************************************************* 20 | 21 | public static void main(String[] args) throws Exception { 22 | ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); 23 | BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); 24 | 25 | DataSet input = env.fromElements( 26 | new WC("Hello", 1), 27 | new WC("Ciao", 1), 28 | new WC("Hello", 1)); 29 | 30 | Table table = tEnv.fromDataSet(input); 31 | 32 | Table filtered = table 33 | .groupBy("word") 34 | .select("word, frequency.sum as frequency") 35 | .filter("frequency = 2"); 36 | 37 | DataSet result = tEnv.toDataSet(filtered, WC.class); 38 | 39 | result.print(); 40 | } 41 | 42 | // ************************************************************************* 43 | // USER DATA TYPES 44 | // ************************************************************************* 45 | 46 | /** 47 | * Simple POJO containing a word and its respective count. 48 | */ 49 | public static class WC { 50 | public String word; 51 | public long frequency; 52 | 53 | // public constructor to make it a Flink POJO 54 | public WC() {} 55 | 56 | public WC(String word, long frequency) { 57 | this.word = word; 58 | this.frequency = frequency; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "WC " + word + " " + frequency; 64 | } 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /flink-example/src/main/java/com/matt/test/worcount/SocketTextStreamWordCount.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.worcount; 2 | 3 | import org.apache.flink.api.common.functions.FlatMapFunction; 4 | import org.apache.flink.api.java.tuple.Tuple2; 5 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 6 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 8 | import org.apache.flink.util.Collector; 9 | 10 | /** 11 | * @author matt 12 | * @date 2019-03-18 20:06 13 | */ 14 | public class SocketTextStreamWordCount { 15 | public static void main(String[] args) throws Exception { 16 | if (args.length != 2) { 17 | System.err.println("USAGE:\nSocketTextStreamWordCount "); 18 | return; 19 | } 20 | 21 | String hostname = args[0]; 22 | Integer port = Integer.parseInt(args[1]); 23 | 24 | // set up the streaming execution environment 25 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 26 | 27 | //获取数据 28 | DataStreamSource stream = env.socketTextStream(hostname, port); 29 | 30 | //计数 31 | SingleOutputStreamOperator> sum = stream.flatMap(new LineSplitter()) 32 | .keyBy(0) 33 | .sum(1); 34 | 35 | sum.print(); 36 | 37 | env.execute("Java WordCount from SocketTextStream Example"); 38 | } 39 | 40 | public static final class LineSplitter implements FlatMapFunction> { 41 | public void flatMap(String s, Collector> collector) { 42 | String[] tokens = s.toLowerCase().split("\\W+"); 43 | 44 | for (String token : tokens) { 45 | if (token.length() > 0) { 46 | collector.collect(new Tuple2(token, 1)); 47 | } 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /flink-example/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=TRACE, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.Threshold=TRACE 5 | log4j.appender.console.ImmediateFlush=true 6 | log4j.appender.console.Target=System.err 7 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.console.layout.ConversionPattern=[%-5p] %d(%r) --> [%t] %l: %m %x %n 9 | 10 | -------------------------------------------------------------------------------- /kafka-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.matt.test 8 | kafka-example 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.2.0 13 | 1.4 14 | 2.8.5 15 | 3.4.14 16 | 17 | 18 | 19 | 20 | org.apache.kafka 21 | kafka-clients 22 | ${kafka-vesion} 23 | 24 | 25 | org.apache.kafka 26 | kafka-streams 27 | ${kafka-vesion} 28 | 29 | 30 | 31 | org.slf4j 32 | slf4j-api 33 | 1.7.21 34 | 35 | 36 | org.slf4j 37 | slf4j-log4j12 38 | 1.7.21 39 | 40 | 41 | 42 | commons-cli 43 | commons-cli 44 | ${cli-version} 45 | 46 | 47 | 48 | com.google.code.gson 49 | gson 50 | ${gson-version} 51 | 52 | 53 | 54 | org.apache.zookeeper 55 | zookeeper 56 | ${zookeeper-version} 57 | 58 | 59 | 60 | 61 | 62 | org.apache.maven.plugins 63 | maven-shade-plugin 64 | 2.4 65 | 66 | 67 | package 68 | 69 | shade 70 | 71 | 72 | ${project.artifactId}-${project.version}-with-dependencies 73 | 74 | 75 | 76 | 77 | 78 | org.apache.maven.plugins 79 | maven-compiler-plugin 80 | 81 | 1.7 82 | 1.7 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /kafka-example/src/main/java/com/matt/test/kafka/producer/ProducerIdempotenceExample.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka.producer; 2 | 3 | import com.google.gson.JsonObject; 4 | 5 | import java.text.SimpleDateFormat; 6 | import java.util.Properties; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.DefaultParser; 10 | import org.apache.commons.cli.HelpFormatter; 11 | import org.apache.commons.cli.Option; 12 | import org.apache.commons.cli.Options; 13 | import org.apache.commons.cli.ParseException; 14 | import org.apache.kafka.clients.producer.KafkaProducer; 15 | import org.apache.kafka.clients.producer.ProducerConfig; 16 | import org.apache.kafka.clients.producer.ProducerRecord; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | /** 21 | * kafka producer example with idempotence, produce data which is in json 22 | */ 23 | public class ProducerIdempotenceExample { 24 | 25 | private static String[] cityType = new String[]{"A", "B", "C", "D", "E", "F", "G"}; 26 | private static final Logger logger = LoggerFactory.getLogger(ProducerIdempotenceExample.class); 27 | 28 | 29 | public static void main(String[] args) { 30 | Options options = new Options(); 31 | Option input = new Option("b", "bootstrap.servers", true, "the bootstrap.servers producer using"); 32 | input.setRequired(true); 33 | options.addOption(input); 34 | Option output = new Option("t", "topic", true, "the topic producer using"); 35 | output.setRequired(true); 36 | options.addOption(output); 37 | DefaultParser parser = new DefaultParser(); 38 | HelpFormatter formatter = new HelpFormatter(); 39 | 40 | try { 41 | CommandLine cmd = parser.parse(options, args); 42 | String e = cmd.getOptionValue("bootstrap.servers"); 43 | String topic = cmd.getOptionValue("topic"); 44 | Properties props = new Properties(); 45 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 46 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 47 | props.put("max.request.size", Integer.valueOf(4194304)); 48 | props.put("batch.size", Integer.valueOf(4194304)); 49 | props.put("retries", Integer.valueOf(3)); 50 | props.put("linger.ms", Integer.valueOf(50)); 51 | props.put("client.id", "ProducerIdempotenceExample"); 52 | props.put("bootstrap.servers", e); 53 | props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); 54 | props.put("acks", "all"); 55 | KafkaProducer producer = new KafkaProducer(props); 56 | int i = 0; 57 | 58 | while (true) { 59 | long timestamp = System.currentTimeMillis(); 60 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 61 | 62 | try { 63 | String e1 = sdf.format(Long.valueOf(timestamp)); 64 | JsonObject msg = new JsonObject(); 65 | msg.addProperty("id", Integer.valueOf(i)); 66 | msg.addProperty("datetime", e1); 67 | msg.addProperty("city_rank", cityType[i % 7]); 68 | producer.send(new ProducerRecord(topic, msg.toString())); 69 | 70 | try { 71 | Thread.sleep(200L); 72 | } catch (InterruptedException var18) { 73 | var18.printStackTrace(); 74 | } 75 | 76 | ++i; 77 | if (i % 100 == 0) { 78 | logger.info("Producer has sent 100 msgs."); 79 | } 80 | } catch (Exception var19) { 81 | var19.printStackTrace(); 82 | logger.error("Forcing producer close!"); 83 | producer.flush(); 84 | producer.close(); 85 | System.exit(0); 86 | } 87 | } 88 | } catch (ParseException var20) { 89 | System.out.println(var20.getMessage()); 90 | formatter.printHelp("utility-name", options); 91 | System.exit(1); 92 | } 93 | } 94 | 95 | } -------------------------------------------------------------------------------- /kafka-example/src/main/java/com/matt/test/kafka/producer/ProducerJsonExample.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka.producer; 2 | 3 | import com.google.gson.JsonObject; 4 | 5 | import java.text.SimpleDateFormat; 6 | import java.util.Properties; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.DefaultParser; 10 | import org.apache.commons.cli.HelpFormatter; 11 | import org.apache.commons.cli.Option; 12 | import org.apache.commons.cli.Options; 13 | import org.apache.commons.cli.ParseException; 14 | import org.apache.kafka.clients.producer.KafkaProducer; 15 | import org.apache.kafka.clients.producer.ProducerRecord; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | /** 20 | * kafka producer example, produce data which is in json 21 | */ 22 | public class ProducerJsonExample { 23 | 24 | private static String[] cityType = new String[]{"A", "B", "C", "D", "E", "F", "G"}; 25 | private static final Logger logger = LoggerFactory.getLogger(ProducerJsonExample.class); 26 | 27 | 28 | public static void main(String[] args) { 29 | Options options = new Options(); 30 | Option input = new Option("b", "bootstrap.servers", true, "the bootstrap.servers producer using"); 31 | input.setRequired(true); 32 | options.addOption(input); 33 | Option output = new Option("t", "topic", true, "the topic producer using"); 34 | output.setRequired(true); 35 | options.addOption(output); 36 | DefaultParser parser = new DefaultParser(); 37 | HelpFormatter formatter = new HelpFormatter(); 38 | 39 | try { 40 | CommandLine cmd = parser.parse(options, args); 41 | String e = cmd.getOptionValue("bootstrap.servers"); 42 | String topic = cmd.getOptionValue("topic"); 43 | Properties props = new Properties(); 44 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 45 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 46 | props.put("max.request.size", Integer.valueOf(4194304)); 47 | props.put("batch.size", Integer.valueOf(4194304)); 48 | props.put("retries", Integer.valueOf(3)); 49 | props.put("linger.ms", Integer.valueOf(50)); 50 | props.put("client.id", "ProducerJsonExample"); 51 | props.put("bootstrap.servers", e); 52 | KafkaProducer producer = new KafkaProducer(props); 53 | int i = 0; 54 | 55 | while (true) { 56 | long timestamp = System.currentTimeMillis(); 57 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 58 | 59 | try { 60 | String e1 = sdf.format(Long.valueOf(timestamp)); 61 | JsonObject msg = new JsonObject(); 62 | msg.addProperty("id", Integer.valueOf(i)); 63 | msg.addProperty("datetime", e1); 64 | msg.addProperty("city_rank", cityType[i % 7]); 65 | producer.send(new ProducerRecord(topic, msg.toString())); 66 | 67 | try { 68 | Thread.sleep(200L); 69 | } catch (InterruptedException var18) { 70 | var18.printStackTrace(); 71 | } 72 | 73 | ++i; 74 | if (i % 100 == 0) { 75 | logger.info("Producer has sent 100 msgs."); 76 | } 77 | } catch (Exception var19) { 78 | var19.printStackTrace(); 79 | logger.error("Forcing producer close!"); 80 | producer.flush(); 81 | producer.close(); 82 | System.exit(0); 83 | } 84 | } 85 | } catch (ParseException var20) { 86 | System.out.println(var20.getMessage()); 87 | formatter.printHelp("utility-name", options); 88 | System.exit(1); 89 | } 90 | } 91 | 92 | } -------------------------------------------------------------------------------- /kafka-example/src/main/java/com/matt/test/kafka/producer/ProducerRecoverTest.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | 5 | import java.lang.reflect.Field; 6 | import java.lang.reflect.InvocationTargetException; 7 | import java.lang.reflect.Method; 8 | import java.util.Properties; 9 | 10 | /** 11 | * @author matt 12 | * @date 2019-05-25 22:54 13 | */ 14 | public class ProducerRecoverTest { 15 | 16 | private static KafkaProducer initKafkaProducer(String transactionalId, String servers) { 17 | Properties props = new Properties(); 18 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 19 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 20 | props.put("max.request.size", Integer.valueOf(4194304)); 21 | props.put("batch.size", Integer.valueOf(4194304)); 22 | props.put("retries", Integer.valueOf(3)); 23 | props.put("linger.ms", Integer.valueOf(50)); 24 | props.put("client.id", "ProducerIdempotenceExample"); 25 | props.put("bootstrap.servers", servers); 26 | props.put("enable.idempotence", "true"); 27 | props.put("transactional.id", transactionalId); 28 | props.put("acks", "all"); 29 | KafkaProducer producer = new KafkaProducer(props); 30 | return producer; 31 | } 32 | 33 | private static Object invoke(Object object, String methodName, Object... args) { 34 | Class[] argTypes = new Class[args.length]; 35 | for (int i = 0; i < args.length; i++) { 36 | argTypes[i] = args[i].getClass(); 37 | } 38 | return invoke(object, methodName, argTypes, args); 39 | } 40 | 41 | private static Object invoke(Object object, String methodName, Class[] argTypes, Object[] args) { 42 | try { 43 | Method method = object.getClass().getDeclaredMethod(methodName, argTypes); 44 | method.setAccessible(true); 45 | return method.invoke(object, args); 46 | } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { 47 | throw new RuntimeException("Incompatible KafkaProducer version", e); 48 | } 49 | } 50 | 51 | //note: 获得某个类的值 52 | private static Object getValue(Object object, String fieldName) { 53 | return getValue(object, object.getClass(), fieldName); 54 | } 55 | 56 | private static Object getValue(Object object, Class clazz, String fieldName) { 57 | try { 58 | Field field = clazz.getDeclaredField(fieldName); 59 | field.setAccessible(true); 60 | return field.get(object); 61 | } catch (NoSuchFieldException | IllegalAccessException e) { 62 | throw new RuntimeException("Incompatible KafkaProducer version", e); 63 | } 64 | } 65 | 66 | //note: 设置相应的 value 67 | private static void setValue(Object object, String fieldName, Object value) { 68 | try { 69 | Field field = object.getClass().getDeclaredField(fieldName); 70 | field.setAccessible(true); 71 | field.set(object, value); 72 | } catch (NoSuchFieldException | IllegalAccessException e) { 73 | throw new RuntimeException("Incompatible KafkaProducer version", e); 74 | } 75 | } 76 | 77 | private static Enum getEnum(String enumFullName) { 78 | String[] x = enumFullName.split("\\.(?=[^\\.]+$)"); 79 | if (x.length == 2) { 80 | String enumClassName = x[0]; 81 | String enumName = x[1]; 82 | try { 83 | Class cl = (Class) Class.forName(enumClassName); 84 | return Enum.valueOf(cl, enumName); 85 | } catch (ClassNotFoundException e) { 86 | throw new RuntimeException("Incompatible KafkaProducer version", e); 87 | } 88 | } 89 | return null; 90 | } 91 | 92 | public static void main(String[] args) { 93 | String transactionalId = args[0]; 94 | String servers = args[1]; 95 | KafkaProducer producer = initKafkaProducer(transactionalId, servers); 96 | 97 | // recover transactionManager 98 | Object transactionManager = getValue(producer, "transactionManager"); 99 | Object sequenceNumbers = getValue(transactionManager, "sequenceNumbers"); 100 | 101 | invoke(transactionManager, "transitionTo", 102 | getEnum("org.apache.kafka.clients.producer.internals.TransactionManager$State.INITIALIZING")); 103 | invoke(sequenceNumbers, "clear"); 104 | 105 | // recover producerIdAndEpoch 106 | long producerId = 100l; 107 | short epoch = 2; 108 | Object producerIdAndEpoch = getValue(transactionManager, "producerIdAndEpoch"); 109 | setValue(producerIdAndEpoch, "producerId", producerId); 110 | setValue(producerIdAndEpoch, "epoch", epoch); 111 | 112 | invoke(transactionManager, "transitionTo", 113 | getEnum("org.apache.kafka.clients.producer.internals.TransactionManager$State.READY")); 114 | 115 | invoke(transactionManager, "transitionTo", 116 | getEnum("org.apache.kafka.clients.producer.internals.TransactionManager$State.IN_TRANSACTION")); 117 | 118 | setValue(transactionManager, "transactionStarted", true); 119 | 120 | System.out.println("KafkaProducer recover success."); 121 | producer.commitTransaction(); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /kafka-example/src/main/java/com/matt/test/kafka/producer/ProducerTransactionExample.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka.producer; 2 | 3 | import com.google.gson.JsonObject; 4 | 5 | import java.text.SimpleDateFormat; 6 | import java.util.Properties; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.DefaultParser; 10 | import org.apache.commons.cli.HelpFormatter; 11 | import org.apache.commons.cli.Option; 12 | import org.apache.commons.cli.Options; 13 | import org.apache.commons.cli.ParseException; 14 | import org.apache.kafka.clients.producer.KafkaProducer; 15 | import org.apache.kafka.clients.producer.ProducerRecord; 16 | import org.apache.kafka.common.KafkaException; 17 | import org.apache.kafka.common.errors.ProducerFencedException; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | /** 22 | * kafka producer example with transaction, produce data which is in json 23 | */ 24 | public class ProducerTransactionExample { 25 | 26 | private static String[] cityType = new String[]{"A", "B", "C", "D", "E", "F", "G"}; 27 | private static final Logger logger = LoggerFactory.getLogger(ProducerTransactionExample.class); 28 | 29 | 30 | public static void main(String[] args) { 31 | Options options = new Options(); 32 | Option input = new Option("b", "bootstrap.servers", true, "the bootstrap.servers producer using"); 33 | input.setRequired(true); 34 | options.addOption(input); 35 | Option output = new Option("t", "topic", true, "the topic producer using"); 36 | output.setRequired(true); 37 | options.addOption(output); 38 | DefaultParser parser = new DefaultParser(); 39 | HelpFormatter formatter = new HelpFormatter(); 40 | 41 | try { 42 | CommandLine cmd = parser.parse(options, args); 43 | String e = cmd.getOptionValue("bootstrap.servers"); 44 | String topic = cmd.getOptionValue("topic"); 45 | Properties props = new Properties(); 46 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 47 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 48 | props.put("max.request.size", Integer.valueOf(4194304)); 49 | props.put("batch.size", Integer.valueOf(4194304)); 50 | props.put("retries", Integer.valueOf(3)); 51 | props.put("linger.ms", Integer.valueOf(50)); 52 | props.put("client.id", "ProducerIdempotenceExample"); 53 | props.put("bootstrap.servers", e); 54 | props.put("enable.idempotence", "true"); 55 | props.put("transactional.id", "test-transactional"); 56 | props.put("acks", "all"); 57 | KafkaProducer producer = new KafkaProducer(props); 58 | int i = 0; 59 | producer.initTransactions(); 60 | 61 | while (true) { 62 | long timestamp = System.currentTimeMillis(); 63 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 64 | 65 | try { 66 | String e1 = sdf.format(Long.valueOf(timestamp)); 67 | JsonObject msg = new JsonObject(); 68 | msg.addProperty("id", Integer.valueOf(i)); 69 | msg.addProperty("datetime", e1); 70 | msg.addProperty("city_rank", cityType[i % 7]); 71 | 72 | try { 73 | producer.beginTransaction(); 74 | producer.send(new ProducerRecord(topic, "0", msg.toString())); 75 | producer.send(new ProducerRecord(topic, "1", msg.toString())); 76 | producer.send(new ProducerRecord(topic, "2", msg.toString())); 77 | producer.commitTransaction(); 78 | } catch (ProducerFencedException var19) { 79 | var19.printStackTrace(); 80 | producer.close(); 81 | } catch (KafkaException var20) { 82 | var20.printStackTrace(); 83 | producer.abortTransaction(); 84 | } 85 | 86 | try { 87 | Thread.sleep(200L); 88 | } catch (InterruptedException var18) { 89 | var18.printStackTrace(); 90 | } 91 | 92 | ++i; 93 | if (i % 100 == 0) { 94 | logger.info("Producer has sent 100 msgs."); 95 | } 96 | } catch (Exception var21) { 97 | var21.printStackTrace(); 98 | logger.error("Forcing producer close!"); 99 | producer.flush(); 100 | producer.close(); 101 | System.exit(0); 102 | } 103 | } 104 | } catch (ParseException var22) { 105 | System.out.println(var22.getMessage()); 106 | formatter.printHelp("utility-name", options); 107 | System.exit(1); 108 | } 109 | } 110 | 111 | } -------------------------------------------------------------------------------- /kafka-example/src/main/java/com/matt/test/kafka/zookeeper/ZooKeeperWatcherTest.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.kafka.zookeeper; 2 | 3 | public class ZooKeeperWatcherTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /kafka-example/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | log4j.rootLogger=INFO,stdout 16 | 17 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 18 | log4j.appender.stdout.Target=System.out 19 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n -------------------------------------------------------------------------------- /leetcode/README.md: -------------------------------------------------------------------------------- 1 | 这个工程是对照极客时间课程《数据结构与算法之美》和 [编程之法：面试与算法心得](https://www.kancloud.cn/kancloud/the-art-of-programming/41632) 做的相应练习，计划每个月一个专题，每周刷一道题（具体的内容会记录在这个文档中），弥补自己算法及数据结构能力的不足。 2 | 3 | > 工作中，发现自己的数据结构设计和算法能力比较薄弱，像比较复杂的算法（比如：动态规划这种）基本没有怎么学习过，正如耗子叔在 [LeetCode编程训练](https://coolshell.cn/articles/12052.html) 中说的一样，LeetCode 对个人的编程能力训练或提高非常有帮助。这里，我的计划并不是把 LeetCode 的题都刷完或者刷多少道，而是为了系统地把数据结构和算法相关的内容练习一下，这里会按系列去练习，每个月选择一个专题，练习4道有代表性的题目，希望能坚持下去。 4 | 5 | ### 第一个月：字符串 6 | 7 | | 序号 | 题目 | 代码链接 | 文章链接 | 时间 | 备注 | 8 | | --- | --- | --- | --- | --- | --- | 9 | | 1 | 旋转字符串 [796 Rotate String](https://leetcode.com/problems/rotate-string/) | | | 2019.3.24 | EASY | 10 | | 2 | 回文字符串 [5 Longest Palindromic Substring](https://leetcode.com/problems/longest-palindromic-substring/) | | | | MEDIUM | 11 | | 3 | [730 Count Different Palindromic Subsequences](https://leetcode.com/problems/count-different-palindromic-subsequences/) | | | | HARD | 12 | | 4 | 全排列问题 [47 Permutations II](https://leetcode.com/problems/permutations-ii/) | | | | MEDIUM | 13 | 14 | > 第四道题，输入的整形数组跟输入一个字符串区别并不大。 15 | 16 | -------------------------------------------------------------------------------- /leetcode/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.matt.test 8 | leetcode 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | junit 14 | junit 15 | 4.13.1 16 | 17 | 18 | 19 | org.testng 20 | testng 21 | 6.8.8 22 | 23 | 24 | -------------------------------------------------------------------------------- /leetcode/src/main/java/com/matt/test/leetcode/string/RotateString.java: -------------------------------------------------------------------------------- 1 | package com.matt.test.leetcode.string; 2 | 3 | /** 4 | * leetcode(796 Rotate String): https://leetcode.com/problems/rotate-string/ 5 | * 6 | * @author matt 7 | * @date 2019-03-24 22:09 8 | */ 9 | public class RotateString { 10 | 11 | public static void main(String[] args) { 12 | assert rotateString("abcde", "cdeab"); 13 | assert !rotateString("abcde", "abced"); 14 | System.out.println(rotateString2(null, null)); 15 | System.out.println(rotateString2("", "")); 16 | 17 | String s = "'b\"a"; 18 | String startQuote = "'"; 19 | String endQuote = "'"; 20 | 21 | System.out.println(s.startsWith(startQuote)); 22 | System.out.println(s.endsWith(endQuote)); 23 | System.out.println(s.substring(1, s.length()-1)); 24 | 25 | assert s.startsWith(startQuote) && s.endsWith(endQuote) : s; 26 | } 27 | 28 | public static boolean rotateString(String A, String B) { 29 | if ((A == null && B == null) || (A.length() == 0 && B.length() == 0)) { 30 | return true; 31 | } 32 | boolean ans = false; 33 | String rotateA = A; 34 | for (int i = 0; i < A.length(); i++) { 35 | char c = A.charAt(i); 36 | rotateA = rotateA.substring(1) + c; 37 | if (rotateA.equals(B)) { 38 | ans = true; 39 | } 40 | } 41 | return ans; 42 | } 43 | 44 | /** 45 | * this solution just one code，very well 46 | * 47 | * @param A 48 | * @param B 49 | * @return 50 | */ 51 | public static boolean rotateString2(String A, String B) { 52 | return (A == null && B == null) || (A.length() == B.length() && (A + A).contains(B)); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.matt.test 6 | program-example 7 | 1.0-SNAPSHOT 8 | pom 9 | 10 | program-example 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | bookkeeper-example 19 | flink-example 20 | kafka-example 21 | calcite-example 22 | 23 | 24 | 25 | 26 | junit 27 | junit 28 | 4.13.1 29 | test 30 | 31 | 32 | 33 | --------------------------------------------------------------------------------