├── .gitignore ├── README.md ├── distributed-components ├── rpc-learning │ ├── .gitignore │ ├── flink │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── runtime │ │ │ └── rpc │ │ │ └── pekko │ │ │ ├── ClientServerRpcExample.java │ │ │ ├── HelloGateway.java │ │ │ └── HelloRpcEndpoint.java │ └── pom.xml └── zookeeper-learning │ ├── .gitignore │ ├── pom.xml │ └── src │ └── main │ └── java │ └── org │ └── apache │ └── zookeeper │ └── learning │ ├── framework │ └── CrudExamples.java │ ├── leader │ ├── election │ │ ├── ExampleClient.java │ │ └── LeaderSelectorExample.java │ └── latch │ │ └── LeaderLatchExample.java │ └── watcher │ └── WatcherExample.java ├── flink-learning ├── .gitignore ├── flink-common │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ ├── apacge │ │ └── flink │ │ │ └── common │ │ │ ├── converter │ │ │ └── JdbcRowConverter.java │ │ │ └── join │ │ │ ├── JdbcRowLookupFunction.java │ │ │ └── RowLookupFunction.java │ │ └── apache │ │ └── flink │ │ └── connector │ │ └── jdbc │ │ └── internal │ │ └── options │ │ └── JdbcLookupOptions.java ├── flink-datastream │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── learning │ │ │ ├── broadcast │ │ │ └── BroadCastExample.java │ │ │ ├── join │ │ │ ├── CustomIntervalJoin.java │ │ │ ├── IntervalJoin.java │ │ │ └── WindowJoin.java │ │ │ ├── state │ │ │ └── MapStateExample.java │ │ │ ├── tool │ │ │ ├── IdPartitioner.java │ │ │ └── KafkaPartitionProducer.java │ │ │ ├── utils │ │ │ ├── Order.java │ │ │ ├── OrderMapper.java │ │ │ ├── Shipment.java │ │ │ └── ShipmentMapper.java │ │ │ └── watermark │ │ │ ├── CustomWatermarkGeneratorExample.java │ │ │ ├── NonSourceWatermarkAssignExample.java │ │ │ ├── SourceWatermarkAssignExample.java │ │ │ ├── SourceWithIdlenessExample.java │ │ │ └── WatermarkAfterFilterExample.java │ │ └── resources │ │ ├── join │ │ └── window_join.txt │ │ └── log4j2.properties ├── flink-sql │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── learning │ │ │ └── table │ │ │ ├── basic │ │ │ └── SQLExample.java │ │ │ ├── conversion │ │ │ ├── LookupJoinExample.java │ │ │ ├── SimpleExample.java │ │ │ └── WatermarkExample.java │ │ │ ├── cube │ │ │ └── SimpleCubeExample.java │ │ │ ├── sink │ │ │ └── SinkModeTest.java │ │ │ └── utils │ │ │ ├── Order.java │ │ │ ├── OrderMapper.java │ │ │ ├── Shipment.java │ │ │ ├── ShipmentMapper.java │ │ │ └── Utils.java │ │ └── resources │ │ └── log4j2.properties ├── flink-udx │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── flink │ │ └── learning │ │ └── udaf │ │ ├── GenericRecord.java │ │ ├── GenericRecordAccumulator.java │ │ └── GenericRecordAgg.java └── pom.xml ├── hadoop-learning ├── hdfs-learning │ ├── dependency-reduced-pom.xml │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── HDFSClientExample.java │ │ └── resources │ │ └── log4j.properties ├── mapreduce-learning │ ├── dependency-reduced-pom.xml │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ ├── PhoneFlowCount.java │ │ └── WordCount.java │ │ └── resources │ │ ├── HTTP_20130313143750.dat │ │ ├── log4j.properties │ │ └── wordcount.txt ├── pom.xml ├── rpc-learning │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ ├── Client.java │ │ ├── MyInterface.java │ │ ├── MyInterfaceImpl.java │ │ └── Server.java └── yarn-learning │ ├── dependency-reduced-pom.xml │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── hadoop │ │ └── yarn │ │ └── applications │ │ └── distributedshell │ │ ├── ApplicationMaster.java │ │ ├── Client.java │ │ ├── DSConstants.java │ │ ├── Log4jPropertyHelper.java │ │ ├── PlacementSpec.java │ │ └── readme.md │ └── resources │ ├── log4j.properties │ └── yarn-site.xml ├── hbase-learning ├── hbase1 │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── hbase │ │ │ └── learning │ │ │ └── filter │ │ │ ├── CustomFilter.java │ │ │ ├── FilterExample.java │ │ │ ├── Test.java │ │ │ ├── TestRowMutations.java │ │ │ └── Util.java │ │ ├── resources │ │ └── hbase-site.xml │ │ └── scala │ │ └── org │ │ └── apache │ │ └── hbase │ │ ├── bulkload │ │ ├── BulkLoadPartitioner.scala │ │ ├── ByteArrayWrapper.scala │ │ └── KeyFamilyQualifier.scala │ │ └── learning │ │ └── HBaseSparkBulkLoad.scala ├── hbase2 │ └── pom.xml └── pom.xml ├── hive-learning ├── pom.xml └── src │ └── main │ └── java │ ├── MyStringLength.java │ ├── MyUDTF.java │ └── format │ ├── CustomTextInputFormat.java │ └── CustomTextOutputFormat.java ├── kafka-learning ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── kafka │ └── learning │ └── transaction │ ├── ConsumerExample.java │ └── ProducerExample.java ├── spark-learning ├── pom.xml └── src │ └── main │ ├── java │ └── JavaWordCount.java │ ├── resources │ ├── inverted_index.txt │ └── log4j.properties │ └── scala │ ├── InvertedIndex.scala │ └── WordCount.scala └── sql ├── antlr-learning ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── antlr │ └── v4 │ └── examples │ └── playdb │ ├── PlayDB.java │ ├── SQLVisitor.java │ ├── SelectStmt.java │ ├── WhereExpr.java │ └── parser │ ├── SQLite.g4 │ ├── SQLite.interp │ ├── SQLite.tokens │ ├── SQLiteBaseListener.java │ ├── SQLiteBaseVisitor.java │ ├── SQLiteLexer.interp │ ├── SQLiteLexer.java │ ├── SQLiteLexer.tokens │ ├── SQLiteListener.java │ ├── SQLiteParser.java │ └── SQLiteVisitor.java ├── calcite-learning ├── README.md ├── calcite-adapter │ ├── calcite-adapter-hbase │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── apache │ │ │ │ │ └── calcite │ │ │ │ │ └── adapter │ │ │ │ │ └── hbase │ │ │ │ │ ├── HBaseEnumerator.java │ │ │ │ │ ├── HBaseProject.java │ │ │ │ │ ├── HBaseProjectTableScanRule.java │ │ │ │ │ ├── HBaseRel.java │ │ │ │ │ ├── HBaseRules.java │ │ │ │ │ ├── HBaseScannableTable.java │ │ │ │ │ ├── HBaseSchema.java │ │ │ │ │ ├── HBaseSchemaFactory.java │ │ │ │ │ ├── HBaseTable.java │ │ │ │ │ ├── HBaseTableScan.java │ │ │ │ │ ├── HBaseToEnumerableConverter.java │ │ │ │ │ └── HBaseTranslatableTable.java │ │ │ └── resources │ │ │ │ ├── model.json │ │ │ │ └── schema.csv │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── calcite │ │ │ └── adapter │ │ │ └── hbase │ │ │ ├── CodeGenTest.java │ │ │ ├── HBaseExample.java │ │ │ └── HBaseTools.java │ └── pom.xml ├── calcite-avatica │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── calcite │ │ └── avatica │ │ └── example │ │ └── simple │ │ ├── Client.java │ │ ├── CustomAvaticaServer.java │ │ └── ServerExample.java ├── calcite-parser │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── codegen │ │ │ ├── config.fmpp │ │ │ ├── data │ │ │ │ └── Parser.tdd │ │ │ └── includes │ │ │ │ ├── compoundIdentifier.ftl │ │ │ │ └── parserImpls.ftl │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── calcite │ │ │ │ ├── example │ │ │ │ ├── CalciteUtil.java │ │ │ │ ├── converter │ │ │ │ │ └── CalciteSQLConverter.java │ │ │ │ ├── optimizer │ │ │ │ │ ├── AbstractConverterExample.java │ │ │ │ │ ├── CalciteRBO.java │ │ │ │ │ ├── HepPlannerExample.java │ │ │ │ │ ├── IterativeMergeJoinExample.java │ │ │ │ │ ├── JoinExample.java │ │ │ │ │ ├── PruningJoinExample.java │ │ │ │ │ ├── RelSetMergeExample.java │ │ │ │ │ ├── TableScanExample.java │ │ │ │ │ ├── VolcanoPlannerExample.java │ │ │ │ │ └── VolcanoPlannerExample1.java │ │ │ │ ├── overall │ │ │ │ │ ├── BindableMain.java │ │ │ │ │ ├── Main.java │ │ │ │ │ ├── Optimizer.java │ │ │ │ │ ├── SimpleDataContext.java │ │ │ │ │ ├── SimpleSchema.java │ │ │ │ │ ├── SimpleTable.java │ │ │ │ │ └── SimpleTableStatistic.java │ │ │ │ ├── parser │ │ │ │ │ ├── CalciteSQLParser.java │ │ │ │ │ └── ddl │ │ │ │ │ │ ├── SqlCreateTable.java │ │ │ │ │ │ └── SqlTableOption.java │ │ │ │ ├── pretty │ │ │ │ │ └── SQLPrettyExample.java │ │ │ │ └── schemas │ │ │ │ │ └── HrClusteredSchema.java │ │ │ │ └── sql │ │ │ │ ├── SqlSubmit.java │ │ │ │ ├── dialect │ │ │ │ └── FlinkSqlDialect.java │ │ │ │ └── pretty │ │ │ │ └── FlinkSqlPrettyWriter.java │ │ └── resources │ │ │ ├── order.csv │ │ │ ├── sample.txt │ │ │ └── user.csv │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── calcite │ │ └── example │ │ └── rel │ │ └── RelBuilderTest.java └── pom.xml └── javacc-learning ├── pom.xml └── src └── main ├── codegen ├── Adder.jj └── Calculator.jj └── java └── javacc └── learning └── calculator ├── Main.java ├── ast ├── CosNode.java ├── ExprNode.java ├── FactorialNode.java ├── Node.java ├── Operator.java ├── SinNode.java ├── TanNode.java ├── TermNode.java ├── UnaryNode.java └── ValueNode.java └── visitor ├── ASTVisitor.java ├── CalculateVisitor.java └── DumpVisitor.java /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | scalastyle-output.xml 3 | .classpath 4 | .idea 5 | .metadata 6 | .settings 7 | .project 8 | .version.properties 9 | filter.properties 10 | logs.zip 11 | target 12 | tmp 13 | *.class 14 | *.iml 15 | *.swp 16 | *.jar 17 | *.zip 18 | *.log 19 | *.pyc 20 | .DS_Store 21 | build-target 22 | flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/avro/ 23 | flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/generated/ 24 | flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/generated/ 25 | flink-runtime-web/web-dashboard/node/ 26 | flink-runtime-web/web-dashboard/node_modules/ 27 | flink-runtime-web/web-dashboard/web/ 28 | flink-python/dist/ 29 | flink-python/build/ 30 | flink-python/pyflink.egg-info/ 31 | flink-python/apache_flink.egg-info/ 32 | flink-python/docs/_build 33 | flink-python/.tox/ 34 | flink-python/dev/download 35 | flink-python/dev/.conda/ 36 | flink-python/dev/log/ 37 | flink-python/dev/.stage.txt 38 | flink-python/.eggs/ 39 | atlassian-ide-plugin.xml 40 | out/ 41 | /docs/api 42 | /docs/content 43 | /docs/.bundle 44 | /docs/.rubydeps 45 | /docs/ruby2/.bundle 46 | /docs/ruby2/.rubydeps 47 | /docs/.jekyll-metadata 48 | *.ipr 49 | *.iws 50 | tools/flink 51 | tools/flink-* 52 | tools/releasing/release 53 | tools/japicmp-output 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository contains some examples for learning data systems. Specifically contains: 2 | + Examples of SQL parsing & optimizing tools, such as Apache Calcite, Antlr. 3 | + Examples of big data computing systems, such as Flink, Spark. 4 | + Examples of big data storage systems, such as Hadoop, HBase, Kafka. -------------------------------------------------------------------------------- /distributed-components/rpc-learning/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | !**/src/main/**/target/ 4 | !**/src/test/**/target/ 5 | 6 | ### IntelliJ IDEA ### 7 | .idea/modules.xml 8 | .idea/jarRepositories.xml 9 | .idea/compiler.xml 10 | .idea/libraries/ 11 | *.iws 12 | *.iml 13 | *.ipr 14 | 15 | ### Eclipse ### 16 | .apt_generated 17 | .classpath 18 | .factorypath 19 | .project 20 | .settings 21 | .springBeans 22 | .sts4-cache 23 | 24 | ### NetBeans ### 25 | /nbproject/private/ 26 | /nbbuild/ 27 | /dist/ 28 | /nbdist/ 29 | /.nb-gradle/ 30 | build/ 31 | !**/src/main/**/build/ 32 | !**/src/test/**/build/ 33 | 34 | ### VS Code ### 35 | .vscode/ 36 | 37 | ### Mac OS ### 38 | .DS_Store -------------------------------------------------------------------------------- /distributed-components/rpc-learning/flink/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | rpc-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | flink 13 | 14 | 15 | 11 16 | 11 17 | UTF-8 18 | 1.19.0 19 | 20 | 21 | 22 | 23 | org.apache.flink 24 | flink-core 25 | ${flink.version} 26 | 27 | 28 | org.apache.flink 29 | flink-rpc-core 30 | ${flink.version} 31 | 32 | 33 | org.apache.flink 34 | flink-rpc-akka 35 | ${flink.version} 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /distributed-components/rpc-learning/flink/src/main/java/org/apache/flink/runtime/rpc/pekko/ClientServerRpcExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.runtime.rpc.pekko; 2 | 3 | import org.apache.flink.configuration.Configuration; 4 | import org.apache.flink.runtime.rpc.RpcService; 5 | 6 | import java.util.Optional; 7 | 8 | public class ClientServerRpcExample { 9 | 10 | private static RpcService rpcService1; 11 | private static RpcService rpcService2; 12 | 13 | public static void open() throws Exception { 14 | rpcService1 = 15 | PekkoRpcServiceUtils.createRemoteRpcService( 16 | new Configuration(), 17 | "localhost", 18 | "0", 19 | null, 20 | Optional.empty()); 21 | rpcService2 = 22 | PekkoRpcServiceUtils.createRemoteRpcService( 23 | new Configuration(), 24 | "localhost", 25 | "0", 26 | null, 27 | Optional.empty()); 28 | } 29 | 30 | public static void close() throws Exception { 31 | if (rpcService1 != null) { 32 | rpcService1.closeAsync().get(); 33 | } 34 | if (rpcService2 != null) { 35 | rpcService2.closeAsync().get(); 36 | } 37 | } 38 | 39 | public static void main(String[] args) throws Exception { 40 | open(); 41 | 42 | HelloRpcEndpoint helloRpcEndpoint = new HelloRpcEndpoint(rpcService1); 43 | helloRpcEndpoint.start(); 44 | 45 | HelloGateway helloGateway = 46 | rpcService2.connect(helloRpcEndpoint.getAddress(), HelloGateway.class).get(); 47 | String result = helloGateway.hello(); 48 | System.out.println(result); 49 | 50 | close(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /distributed-components/rpc-learning/flink/src/main/java/org/apache/flink/runtime/rpc/pekko/HelloGateway.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.runtime.rpc.pekko; 2 | 3 | import org.apache.flink.runtime.rpc.RpcGateway; 4 | 5 | public interface HelloGateway extends RpcGateway { 6 | 7 | String hello(); 8 | } 9 | -------------------------------------------------------------------------------- /distributed-components/rpc-learning/flink/src/main/java/org/apache/flink/runtime/rpc/pekko/HelloRpcEndpoint.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.runtime.rpc.pekko; 2 | 3 | import org.apache.flink.runtime.rpc.RpcEndpoint; 4 | import org.apache.flink.runtime.rpc.RpcService; 5 | 6 | public class HelloRpcEndpoint extends RpcEndpoint implements HelloGateway { 7 | 8 | public HelloRpcEndpoint(RpcService rpcService) { 9 | super(rpcService); 10 | } 11 | 12 | @Override 13 | public String hello() { 14 | return "Hello"; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /distributed-components/rpc-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | rpc-learning 9 | 1.0-SNAPSHOT 10 | pom 11 | 12 | flink 13 | 14 | 15 | 16 | 11 17 | 11 18 | UTF-8 19 | 20 | 21 | -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | !**/src/main/**/target/ 4 | !**/src/test/**/target/ 5 | 6 | ### IntelliJ IDEA ### 7 | .idea/modules.xml 8 | .idea/jarRepositories.xml 9 | .idea/compiler.xml 10 | .idea/libraries/ 11 | *.iws 12 | *.iml 13 | *.ipr 14 | 15 | ### Eclipse ### 16 | .apt_generated 17 | .classpath 18 | .factorypath 19 | .project 20 | .settings 21 | .springBeans 22 | .sts4-cache 23 | 24 | ### NetBeans ### 25 | /nbproject/private/ 26 | /nbbuild/ 27 | /dist/ 28 | /nbdist/ 29 | /.nb-gradle/ 30 | build/ 31 | !**/src/main/**/build/ 32 | !**/src/test/**/build/ 33 | 34 | ### VS Code ### 35 | .vscode/ 36 | 37 | ### Mac OS ### 38 | .DS_Store -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.apache.zookeeper.learning 8 | zookeeper-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 5.6.0 13 | 14 | 15 | 16 | 17 | org.apache.curator 18 | curator-recipes 19 | ${curator.version} 20 | 21 | 22 | 23 | org.apache.curator 24 | curator-test 25 | ${curator.version} 26 | 27 | 28 | 29 | org.apache.curator 30 | curator-x-discovery 31 | ${curator.version} 32 | 33 | 34 | 35 | org.apache.curator 36 | curator-x-async 37 | ${curator.version} 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/src/main/java/org/apache/zookeeper/learning/leader/election/ExampleClient.java: -------------------------------------------------------------------------------- 1 | package org.apache.zookeeper.learning.leader.election; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.util.concurrent.TimeUnit; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | import org.apache.curator.framework.CuratorFramework; 8 | import org.apache.curator.framework.recipes.leader.LeaderSelector; 9 | import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter; 10 | 11 | /** 12 | * An example leader selector client. Note that {@link LeaderSelectorListenerAdapter} which 13 | * has the recommended handling for connection state issues 14 | */ 15 | public class ExampleClient extends LeaderSelectorListenerAdapter implements Closeable { 16 | private final String name; 17 | private final LeaderSelector leaderSelector; 18 | private final AtomicInteger leaderCount = new AtomicInteger(); 19 | 20 | public ExampleClient(CuratorFramework client, String path, String name) { 21 | this.name = name; 22 | 23 | // create a leader selector using the given path for management 24 | // all participants in a given leader selection must use the same path 25 | // ExampleClient here is also a LeaderSelectorListener but this isn't required 26 | leaderSelector = new LeaderSelector(client, path, this); 27 | 28 | // for most cases you will want your instance to requeue when it relinquishes leadership 29 | leaderSelector.autoRequeue(); 30 | } 31 | 32 | public void start() throws IOException { 33 | // the selection for this instance doesn't start until the leader selector is started 34 | // leader selection is done in the background so this call to leaderSelector.start() returns immediately 35 | leaderSelector.start(); 36 | } 37 | 38 | @Override 39 | public void close() throws IOException { 40 | leaderSelector.close(); 41 | } 42 | 43 | @Override 44 | public void takeLeadership(CuratorFramework client) throws Exception { 45 | // we are now the leader. This method should not return until we want to relinquish leadership 46 | 47 | final int waitSeconds = (int) (5 * Math.random()) + 1; 48 | 49 | System.out.println(name + " is now the leader. Waiting " + waitSeconds + " seconds..."); 50 | System.out.println(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before."); 51 | try { 52 | Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds)); 53 | } catch (InterruptedException e) { 54 | System.err.println(name + " was interrupted."); 55 | Thread.currentThread().interrupt(); 56 | } finally { 57 | System.out.println(name + " relinquishing leadership.\n"); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/src/main/java/org/apache/zookeeper/learning/leader/election/LeaderSelectorExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.zookeeper.learning.leader.election; 2 | 3 | import com.google.common.collect.Lists; 4 | import java.io.BufferedReader; 5 | import java.io.InputStreamReader; 6 | import java.util.List; 7 | import org.apache.curator.framework.CuratorFramework; 8 | import org.apache.curator.framework.CuratorFrameworkFactory; 9 | import org.apache.curator.retry.ExponentialBackoffRetry; 10 | import org.apache.curator.test.TestingServer; 11 | import org.apache.curator.utils.CloseableUtils; 12 | 13 | public class LeaderSelectorExample { 14 | private static final int CLIENT_QTY = 10; 15 | 16 | private static final String PATH = "/examples/leader"; 17 | 18 | public static void main(String[] args) throws Exception { 19 | // all of the useful sample code is in ExampleClient.java 20 | 21 | System.out.println( 22 | "Create " + CLIENT_QTY 23 | + " clients, have each negotiate for leadership and then wait a random number of seconds before letting another leader election occur."); 24 | System.out.println( 25 | "Notice that leader election is fair: all clients will become leader and will do so the same number of times."); 26 | 27 | List clients = Lists.newArrayList(); 28 | List examples = Lists.newArrayList(); 29 | TestingServer server = new TestingServer(); 30 | try { 31 | for (int i = 0; i < CLIENT_QTY; ++i) { 32 | CuratorFramework client = CuratorFrameworkFactory.newClient( 33 | server.getConnectString(), new ExponentialBackoffRetry(1000, 3)); 34 | clients.add(client); 35 | 36 | ExampleClient example = new ExampleClient(client, PATH, "Client #" + i); 37 | examples.add(example); 38 | 39 | client.start(); 40 | example.start(); 41 | } 42 | 43 | System.out.println("Press enter/return to quit\n"); 44 | new BufferedReader(new InputStreamReader(System.in)).readLine(); 45 | } finally { 46 | System.out.println("Shutting down..."); 47 | 48 | for (ExampleClient exampleClient : examples) { 49 | CloseableUtils.closeQuietly(exampleClient); 50 | } 51 | for (CuratorFramework client : clients) { 52 | CloseableUtils.closeQuietly(client); 53 | } 54 | 55 | CloseableUtils.closeQuietly(server); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/src/main/java/org/apache/zookeeper/learning/leader/latch/LeaderLatchExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.zookeeper.learning.leader.latch; 2 | 3 | import org.apache.curator.RetryPolicy; 4 | import org.apache.curator.framework.CuratorFramework; 5 | import org.apache.curator.framework.CuratorFrameworkFactory; 6 | import org.apache.curator.framework.recipes.leader.LeaderLatch; 7 | import org.apache.curator.framework.recipes.leader.LeaderLatchListener; 8 | import org.apache.curator.retry.ExponentialBackoffRetry; 9 | 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.Executors; 12 | 13 | public class LeaderLatchExample { 14 | 15 | private static final String PATH = "/examples/leader"; 16 | 17 | private static final Integer CLIENT_COUNT = 5; 18 | 19 | public static void main(String[] args) throws Exception { 20 | ExecutorService service = Executors.newFixedThreadPool(CLIENT_COUNT); 21 | 22 | for (int i = 0; i < CLIENT_COUNT ; i++) { 23 | final int index = i; 24 | service.submit(() -> { 25 | try { 26 | schedule(index); 27 | } catch (Exception e) { 28 | e.printStackTrace(); 29 | } 30 | }); 31 | } 32 | 33 | Thread.sleep(30 * 1000); 34 | service.shutdownNow(); 35 | } 36 | 37 | private static void schedule(int thread) throws Exception { 38 | CuratorFramework client = getClient(thread); 39 | 40 | LeaderLatch latch = new LeaderLatch(client, PATH, String.valueOf(thread)); 41 | 42 | latch.addListener(new LeaderLatchListener() { 43 | 44 | @Override 45 | public void notLeader() { 46 | System.out.println("Client [" + thread + "] I am the follower !"); 47 | } 48 | 49 | @Override 50 | public void isLeader() { 51 | System.out.println("Client [" + thread + "] I am the leader !"); 52 | } 53 | }); 54 | 55 | latch.start(); 56 | 57 | Thread.sleep(2 * (thread + 5) * 1000); 58 | 59 | latch.close(LeaderLatch.CloseMode.NOTIFY_LEADER); 60 | client.close(); 61 | System.out.println("Client [" + latch.getId() + "] Server closed..."); 62 | } 63 | 64 | private static CuratorFramework getClient(final int thread) { 65 | RetryPolicy rp = new ExponentialBackoffRetry(1000, 3); 66 | 67 | CuratorFramework client = CuratorFrameworkFactory.builder() 68 | .connectString("localhost:2181") 69 | .sessionTimeoutMs(1000000) 70 | .connectionTimeoutMs(3000) 71 | .retryPolicy(rp) 72 | .build(); 73 | client.start(); 74 | System.out.println("Client [" + thread + "] Server connected..."); 75 | return client; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /distributed-components/zookeeper-learning/src/main/java/org/apache/zookeeper/learning/watcher/WatcherExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.zookeeper.learning.watcher; 2 | 3 | import org.apache.curator.framework.CuratorFramework; 4 | import org.apache.curator.framework.CuratorFrameworkFactory; 5 | import org.apache.curator.retry.ExponentialBackoffRetry; 6 | import org.apache.zookeeper.WatchedEvent; 7 | import org.apache.zookeeper.Watcher; 8 | import org.apache.zookeeper.data.Stat; 9 | 10 | public class WatcherExample { 11 | 12 | public static void main(String[] args) throws Exception { 13 | String connectionString = "localhost:2181"; 14 | ExponentialBackoffRetry retryPolicy = 15 | new ExponentialBackoffRetry(1000, 3); 16 | CuratorFramework client = CuratorFrameworkFactory.newClient(connectionString, retryPolicy); 17 | client.start(); 18 | 19 | String workerPath = "/test/listener/remoteNode"; 20 | String subWorkerPath = "/test/listener/remoteNode/id-"; 21 | 22 | Stat stat = client.checkExists().forPath(workerPath); 23 | if (stat == null) { 24 | client.create().creatingParentsIfNeeded().forPath(workerPath); 25 | } 26 | 27 | Watcher watcher = new Watcher() { 28 | @Override 29 | public void process(WatchedEvent watchedEvent) { 30 | System.out.println("Received watched event: " + watchedEvent); 31 | } 32 | }; 33 | byte[] content = client.getData().usingWatcher(watcher).forPath(workerPath); 34 | System.out.println("Content: " + new String(content)); 35 | 36 | client.setData().forPath(workerPath, "1".getBytes()); 37 | client.setData().forPath(workerPath, "2".getBytes()); 38 | 39 | client.close(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flink-learning/.gitignore: -------------------------------------------------------------------------------- 1 | dependency-reduced-pom.xml -------------------------------------------------------------------------------- /flink-learning/flink-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | flink-learning 7 | org.apache.flink.learning 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | flink-common 13 | 14 | 15 | 16 | 17 | org.apache.flink 18 | flink-core 19 | 20 | 21 | org.apache.flink 22 | flink-streaming-java 23 | 24 | 25 | org.apache.flink 26 | flink-clients 27 | 28 | 29 | 30 | 31 | org.apache.flink 32 | flink-rpc-core 33 | ${flink.version} 34 | 35 | 36 | org.apache.flink 37 | flink-rpc-akka 38 | ${flink.version} 39 | 40 | 41 | 42 | 43 | org.apache.flink 44 | flink-table-api-java 45 | 46 | 47 | org.apache.flink 48 | flink-table-api-java-bridge 49 | 50 | 51 | org.apache.flink 52 | flink-table-planner_${scala.binary.version} 53 | 54 | 55 | 56 | 57 | org.apache.flink 58 | flink-connector-jdbc 59 | 60 | 61 | 62 | mysql 63 | mysql-connector-java 64 | 8.0.33 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /flink-learning/flink-common/src/main/java/org/apacge/flink/common/converter/JdbcRowConverter.java: -------------------------------------------------------------------------------- 1 | package org.apacge.flink.common.converter; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | import java.io.Serializable; 6 | import java.sql.PreparedStatement; 7 | import java.sql.ResultSet; 8 | import java.sql.ResultSetMetaData; 9 | import java.sql.SQLException; 10 | 11 | public class JdbcRowConverter implements Serializable { 12 | 13 | public Row toInternal(ResultSet resultSet) throws SQLException { 14 | ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); 15 | int arity = resultSetMetaData.getColumnCount(); 16 | Row row = new Row(arity); 17 | for (int i = 0; i < arity; ++i) { 18 | row.setField(i, resultSet.getObject(i + 1)); 19 | } 20 | return row; 21 | } 22 | 23 | PreparedStatement toExternal(Row row, PreparedStatement preparedStatement) { 24 | return null; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /flink-learning/flink-common/src/main/java/org/apacge/flink/common/join/RowLookupFunction.java: -------------------------------------------------------------------------------- 1 | package org.apacge.flink.common.join; 2 | 3 | import org.apache.flink.streaming.api.functions.ProcessFunction; 4 | import org.apache.flink.types.Row; 5 | 6 | public abstract class RowLookupFunction extends ProcessFunction { 7 | 8 | protected boolean isOuterJoin; 9 | } 10 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | flink-learning 7 | org.apache.flink.learning 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | flink-datastream 13 | 14 | 15 | 16 | org.apache.flink 17 | flink-core 18 | 19 | 20 | org.apache.flink 21 | flink-streaming-java 22 | 23 | 24 | org.apache.flink 25 | flink-clients 26 | 27 | 28 | 29 | org.apache.flink 30 | flink-connector-kafka 31 | 32 | 33 | 34 | 35 | 36 | org.apache.logging.log4j 37 | log4j-slf4j-impl 38 | 39 | 40 | 41 | org.apache.logging.log4j 42 | log4j-api 43 | 44 | 45 | 46 | org.apache.logging.log4j 47 | log4j-core 48 | 49 | 50 | 51 | 52 | org.apache.logging.log4j 53 | log4j-1.2-api 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/join/CustomIntervalJoin.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.join; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.learning.utils.Order; 5 | import org.apache.flink.learning.utils.OrderMapper; 6 | import org.apache.flink.learning.utils.Shipment; 7 | import org.apache.flink.learning.utils.ShipmentMapper; 8 | import org.apache.flink.streaming.api.datastream.DataStream; 9 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 10 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction; 11 | import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction; 12 | import org.apache.flink.util.Collector; 13 | 14 | public class CustomIntervalJoin { 15 | 16 | public static void main(String[] args) { 17 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 18 | env.setParallelism(1); 19 | 20 | DataStream order = env 21 | .socketTextStream("localhost", 8888) 22 | .flatMap(new OrderMapper()) 23 | .assignTimestampsAndWatermarks(WatermarkStrategy 24 | .forMonotonousTimestamps() 25 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 26 | 27 | DataStream shipment = env 28 | .socketTextStream("localhost", 9999) 29 | .flatMap(new ShipmentMapper()) 30 | .assignTimestampsAndWatermarks(WatermarkStrategy 31 | .forMonotonousTimestamps() 32 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 33 | 34 | order 35 | .connect(shipment) 36 | .keyBy(Order::getOrderId, Shipment::getShipId) 37 | .process(new CoProcessFunction() { 38 | @Override 39 | public void processElement1(Order order, CoProcessFunction.Context context, Collector collector) throws Exception { 40 | 41 | } 42 | 43 | @Override 44 | public void processElement2(Shipment shipment, CoProcessFunction.Context context, Collector collector) throws Exception { 45 | 46 | } 47 | }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/join/IntervalJoin.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.join; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.api.java.tuple.Tuple7; 5 | import org.apache.flink.learning.utils.Order; 6 | import org.apache.flink.learning.utils.OrderMapper; 7 | import org.apache.flink.learning.utils.Shipment; 8 | import org.apache.flink.learning.utils.ShipmentMapper; 9 | import org.apache.flink.streaming.api.datastream.DataStream; 10 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 11 | import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction; 12 | import org.apache.flink.streaming.api.windowing.time.Time; 13 | import org.apache.flink.util.Collector; 14 | 15 | public class IntervalJoin { 16 | 17 | public static void main(String[] args) throws Exception { 18 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 19 | env.setParallelism(1); 20 | 21 | DataStream order = env 22 | .socketTextStream("localhost", 8888) 23 | .flatMap(new OrderMapper()) 24 | .assignTimestampsAndWatermarks(WatermarkStrategy 25 | .forMonotonousTimestamps() 26 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 27 | 28 | DataStream shipment = env 29 | .socketTextStream("localhost", 9999) 30 | .flatMap(new ShipmentMapper()) 31 | .assignTimestampsAndWatermarks(WatermarkStrategy 32 | .forMonotonousTimestamps() 33 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 34 | 35 | order.keyBy(Order::getOrderId) 36 | .intervalJoin(shipment.keyBy(Shipment::getOrderId)) 37 | .between(Time.seconds(0), Time.seconds(5)) 38 | .process(new ProcessJoinFunction>() { 39 | @Override 40 | public void processElement(Order o, 41 | Shipment s, 42 | Context context, 43 | Collector> collector) throws Exception { 44 | collector.collect(new Tuple7<>(o.getOrderId(), o.getUserName(), o.getItem(), o.getTimestamp(), 45 | s.getShipId(), s.getCompany(), s.getTimestamp())); 46 | } 47 | }) 48 | .print(); 49 | 50 | env.execute(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/join/WindowJoin.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.join; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.api.common.functions.JoinFunction; 5 | import org.apache.flink.api.java.tuple.Tuple7; 6 | import org.apache.flink.learning.utils.Order; 7 | import org.apache.flink.learning.utils.OrderMapper; 8 | import org.apache.flink.learning.utils.Shipment; 9 | import org.apache.flink.learning.utils.ShipmentMapper; 10 | import org.apache.flink.streaming.api.datastream.DataStream; 11 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 12 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 13 | import org.apache.flink.streaming.api.windowing.time.Time; 14 | 15 | public class WindowJoin { 16 | 17 | public static void main(String[] args) throws Exception { 18 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 19 | env.setParallelism(1); 20 | 21 | DataStream order = env 22 | .socketTextStream("localhost", 8888) 23 | .flatMap(new OrderMapper()) 24 | .assignTimestampsAndWatermarks(WatermarkStrategy 25 | .forMonotonousTimestamps() 26 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 27 | 28 | DataStream shipment = env 29 | .socketTextStream("localhost", 9999) 30 | .flatMap(new ShipmentMapper()) 31 | .assignTimestampsAndWatermarks(WatermarkStrategy 32 | .forMonotonousTimestamps() 33 | .withTimestampAssigner((event, time) -> event.getTimestamp())); 34 | 35 | order.join(shipment) 36 | .where(Order::getOrderId) 37 | .equalTo(Shipment::getOrderId) 38 | .window(TumblingEventTimeWindows.of(Time.seconds(5))) 39 | .apply(new JoinFunction>() { 40 | @Override 41 | public Tuple7 join(Order o, Shipment s) throws Exception { 42 | return new Tuple7<>(o.getOrderId(), o.getUserName(), o.getItem(), o.getTimestamp(), 43 | s.getShipId(), s.getCompany(), s.getTimestamp()); 44 | } 45 | }) 46 | .print(); 47 | 48 | env.execute(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/tool/IdPartitioner.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.tool; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | 6 | import java.util.Map; 7 | 8 | public class IdPartitioner implements Partitioner { 9 | 10 | @Override 11 | public int partition(String topic, 12 | Object key, 13 | byte[] keyBytes, 14 | Object value, 15 | byte[] valueBytes, 16 | Cluster cluster) { 17 | return Integer.parseInt(key.toString()); 18 | } 19 | 20 | @Override 21 | public void close() { 22 | 23 | } 24 | 25 | @Override 26 | public void configure(Map map) { 27 | 28 | } 29 | } -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/tool/KafkaPartitionProducer.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.tool; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | 8 | import java.util.Properties; 9 | import java.util.Scanner; 10 | 11 | public class KafkaPartitionProducer { 12 | 13 | public static void main(String[] args) { 14 | String topic = "test1"; 15 | 16 | Properties props = new Properties(); 17 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 18 | props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName()); 19 | props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName()); 20 | props.setProperty(ProducerConfig.PARTITIONER_CLASS_CONFIG, IdPartitioner.class.getCanonicalName()); 21 | 22 | KafkaProducer producer = new KafkaProducer<>(props); 23 | Scanner scanner = new Scanner(System.in); 24 | while (scanner.hasNextLine()) { 25 | String line = scanner.nextLine(); 26 | String[] items = line.split(","); 27 | String id = items[0]; 28 | producer.send(new ProducerRecord<>(topic, id, line)); 29 | } 30 | producer.close(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/utils/Order.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.utils; 2 | 3 | public class Order { 4 | 5 | private String orderId; 6 | private String userName; 7 | private String item; 8 | private long timestamp; 9 | 10 | public Order() { } 11 | 12 | public Order(String orderId, String userName, String item, long timestamp) { 13 | this.orderId = orderId; 14 | this.userName = userName; 15 | this.item = item; 16 | this.timestamp = timestamp; 17 | } 18 | 19 | public String getOrderId() { 20 | return orderId; 21 | } 22 | 23 | public void setOrderId(String orderId) { 24 | this.orderId = orderId; 25 | } 26 | 27 | public String getUserName() { 28 | return userName; 29 | } 30 | 31 | public void setUserName(String userName) { 32 | this.userName = userName; 33 | } 34 | 35 | public String getItem() { 36 | return item; 37 | } 38 | 39 | public void setItem(String item) { 40 | this.item = item; 41 | } 42 | 43 | public long getTimestamp() { 44 | return timestamp; 45 | } 46 | 47 | public void setTimestamp(long timestamp) { 48 | this.timestamp = timestamp; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/utils/OrderMapper.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.utils; 2 | 3 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.util.Collector; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.LocalDateTime; 10 | import java.time.ZoneOffset; 11 | import java.time.format.DateTimeFormatter; 12 | 13 | public class OrderMapper extends RichFlatMapFunction { 14 | 15 | private static Logger LOGGER = LoggerFactory.getLogger(OrderMapper.class); 16 | 17 | private DateTimeFormatter formatter; 18 | 19 | @Override 20 | public void open(Configuration parameters) throws Exception { 21 | this.formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 22 | } 23 | 24 | @Override 25 | public void flatMap(String line, Collector collector) throws Exception { 26 | try { 27 | String[] items = line.split(","); 28 | String orderId = items[0]; 29 | String userName = items[1]; 30 | String item = items[2]; 31 | long timestamp = LocalDateTime.parse(items[3], formatter).toInstant(ZoneOffset.of("+8")).toEpochMilli(); 32 | collector.collect(new Order(orderId, userName, item, timestamp)); 33 | } catch (Exception e) { 34 | LOGGER.error("Cannot parse line: {}", line); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/utils/Shipment.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.utils; 2 | 3 | public class Shipment { 4 | 5 | private String shipId; 6 | private String orderId; 7 | private String company; 8 | private long timestamp; 9 | 10 | public Shipment(String shipId, String orderId, String company, long timestamp) { 11 | this.shipId = shipId; 12 | this.orderId = orderId; 13 | this.company = company; 14 | this.timestamp = timestamp; 15 | } 16 | 17 | public String getShipId() { 18 | return shipId; 19 | } 20 | 21 | public void setShipId(String shipId) { 22 | this.shipId = shipId; 23 | } 24 | 25 | public String getOrderId() { 26 | return orderId; 27 | } 28 | 29 | public void setOrderId(String orderId) { 30 | this.orderId = orderId; 31 | } 32 | 33 | public String getCompany() { 34 | return company; 35 | } 36 | 37 | public void setCompany(String company) { 38 | this.company = company; 39 | } 40 | 41 | public long getTimestamp() { 42 | return timestamp; 43 | } 44 | 45 | public void setTimestamp(long timestamp) { 46 | this.timestamp = timestamp; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/utils/ShipmentMapper.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.utils; 2 | 3 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.util.Collector; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.LocalDateTime; 10 | import java.time.ZoneOffset; 11 | import java.time.format.DateTimeFormatter; 12 | 13 | public class ShipmentMapper extends RichFlatMapFunction { 14 | 15 | private static Logger LOGGER = LoggerFactory.getLogger(ShipmentMapper.class); 16 | 17 | private DateTimeFormatter formatter; 18 | 19 | @Override 20 | public void open(Configuration parameters) throws Exception { 21 | this.formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 22 | } 23 | 24 | @Override 25 | public void flatMap(String line, Collector collector) throws Exception { 26 | try { 27 | String[] items = line.split(","); 28 | String shipId = items[0]; 29 | String orderId = items[1]; 30 | String company = items[2]; 31 | long timestamp = LocalDateTime.parse(items[3], formatter).toInstant(ZoneOffset.of("+8")).toEpochMilli(); 32 | collector.collect(new Shipment(shipId, orderId, company, timestamp)); 33 | } catch (Exception e) { 34 | LOGGER.error("Cannot parse line: {}", line); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/watermark/CustomWatermarkGeneratorExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.watermark; 2 | 3 | import org.apache.flink.api.common.eventtime.Watermark; 4 | import org.apache.flink.api.common.eventtime.WatermarkGenerator; 5 | import org.apache.flink.api.common.eventtime.WatermarkOutput; 6 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 7 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 8 | import org.apache.flink.connector.kafka.source.KafkaSource; 9 | import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; 10 | import org.apache.flink.streaming.api.datastream.DataStream; 11 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 12 | 13 | import java.time.Duration; 14 | import java.util.Comparator; 15 | import java.util.HashMap; 16 | import java.util.Map; 17 | import java.util.Optional; 18 | 19 | public class CustomWatermarkGeneratorExample { 20 | 21 | public static void main(String[] args) { 22 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 23 | env.setParallelism(1); 24 | env.getConfig().setAutoWatermarkInterval(1); 25 | 26 | KafkaSource source = KafkaSource.builder() 27 | .setBootstrapServers("localhost:9092") 28 | .setTopics("test1") 29 | .setGroupId("my-group") 30 | .setStartingOffsets(OffsetsInitializer.earliest()) 31 | .setValueOnlyDeserializer(new SimpleStringSchema()) 32 | .build(); 33 | 34 | // DataStream streamSource = env.fromSource( 35 | // source, 36 | // WatermarkStrategy.forGenerator( 37 | // (ctx) -> new CustomWatermarkGenerator<>(Duration.ofSeconds(5))), 38 | // "kafka-source"); 39 | } 40 | 41 | public static class VisitWatermarkGenerator implements WatermarkGenerator { 42 | 43 | private final long outOfOrdernessMillis; 44 | private final Map maxTimePerServer = new HashMap<>(2); 45 | 46 | public VisitWatermarkGenerator(Duration maxOutOfOrderness) { 47 | this.outOfOrdernessMillis = maxOutOfOrderness.toMillis(); 48 | } 49 | 50 | @Override 51 | public void onEvent(Visit event, long eventTimestamp, WatermarkOutput output) { 52 | String server = event.serverId; 53 | if (!maxTimePerServer.containsKey(server) || eventTimestamp > maxTimePerServer.get(server)) { 54 | maxTimePerServer.put(server, eventTimestamp); 55 | } 56 | } 57 | 58 | @Override 59 | public void onPeriodicEmit(WatermarkOutput output) { 60 | Optional maxTimestamp = maxTimePerServer.values().stream() 61 | .min(Comparator.comparingLong(Long::valueOf)); 62 | maxTimestamp.ifPresent( 63 | t -> output.emitWatermark(new Watermark(t - outOfOrdernessMillis - 1))); 64 | } 65 | } 66 | 67 | public static class Visit { 68 | public String serverId; 69 | public String userId; 70 | public long eventTime; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/java/org/apache/flink/learning/watermark/WatermarkAfterFilterExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.watermark; 2 | 3 | import cn.hutool.core.date.LocalDateTimeUtil; 4 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 5 | import org.apache.flink.api.common.functions.FlatMapFunction; 6 | import org.apache.flink.api.java.tuple.Tuple2; 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 8 | import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; 9 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 10 | import org.apache.flink.streaming.api.windowing.time.Time; 11 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 12 | import org.apache.flink.util.Collector; 13 | 14 | /** 15 | * 16 | * @author Yu Liebing 17 | * */ 18 | public class WatermarkAfterFilterExample { 19 | 20 | public static void main(String[] args) throws Exception { 21 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 22 | env.setParallelism(1); 23 | env.getConfig().setAutoWatermarkInterval(1); 24 | 25 | env.socketTextStream("localhost", 8899) 26 | .flatMap(new FlatMapFunction>() { 27 | @Override 28 | public void flatMap(String s, Collector> collector) throws Exception { 29 | try { 30 | String[] items = s.split(","); 31 | int id = Integer.parseInt(items[0]); 32 | long ts = Long.parseLong(items[1]); 33 | collector.collect(new Tuple2<>(id, ts)); 34 | } catch (Exception e) { 35 | System.out.println("Error"); 36 | } 37 | } 38 | }) 39 | .assignTimestampsAndWatermarks(WatermarkStrategy 40 | .>forMonotonousTimestamps() 41 | .withTimestampAssigner((t, time) -> t.f1)) 42 | .filter(t -> t.f0 == 1) 43 | .windowAll(TumblingEventTimeWindows.of(Time.seconds(5))) 44 | .apply(new AllWindowFunction, Object, TimeWindow>() { 45 | @Override 46 | public void apply(TimeWindow timeWindow, Iterable> iterable, Collector collector) throws Exception { 47 | System.out.println("========"); 48 | System.out.println("Window range: [" 49 | + LocalDateTimeUtil.format(LocalDateTimeUtil.of(timeWindow.getStart()), "yyyy-MM-dd HH:mm:ss") 50 | + ", " 51 | + LocalDateTimeUtil.format(LocalDateTimeUtil.of(timeWindow.getEnd()), "yyyy-MM-dd HH:mm:ss") 52 | + "]"); 53 | System.out.println("Window items:"); 54 | iterable.forEach(System.out::println); 55 | } 56 | }); 57 | env.execute(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/resources/join/window_join.txt: -------------------------------------------------------------------------------- 1 | order stream: 2 | 1,John,Shoes,2021-10-14 16:00:00 3 | 2,Mike,Hat,2021-10-14 16:00:03 4 | 3,Jack,Shoes,2021-10-14 16:00:06 5 | 6 | shipment stream 7 | 101,1,ZhongTong,2021-10-14 16:00:01 8 | 102,2,YuanTong,2021-10-14 16:00:04 9 | 103,3,ShunFeng,2021-10-14 16:00:07 -------------------------------------------------------------------------------- /flink-learning/flink-datastream/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | rootLogger.level = INFO 2 | rootLogger.appenderRef.console.ref = ConsoleAppender 3 | 4 | appender.console.name = ConsoleAppender 5 | appender.console.type = CONSOLE 6 | appender.console.layout.type = PatternLayout 7 | appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n 8 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/basic/SQLExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.basic; 2 | 3 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 4 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 5 | 6 | public class SQLExample { 7 | 8 | public static void main(String[] args) { 9 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 10 | StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/conversion/SimpleExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.conversion; 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation; 4 | import org.apache.flink.api.common.typeinfo.Types; 5 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 6 | import org.apache.flink.streaming.api.datastream.DataStream; 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 8 | import org.apache.flink.streaming.api.functions.ProcessFunction; 9 | import org.apache.flink.table.api.Table; 10 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 11 | import org.apache.flink.types.Row; 12 | import org.apache.flink.util.Collector; 13 | 14 | /** 15 | * A simple example shows how to convert between Table & DataStream 16 | */ 17 | public class SimpleExample { 18 | 19 | public static void main(String[] args) throws Exception { 20 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 21 | StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); 22 | 23 | DataStream dataStream = env.fromElements( 24 | Row.of("Alice", 12), 25 | Row.of("Bob", 10), 26 | Row.of("Alice", 100)); 27 | 28 | RowTypeInfo rowTypeInfo = new RowTypeInfo( 29 | new TypeInformation[]{Types.STRING, Types.INT}, 30 | new String[]{"name", "score"}); 31 | DataStream processedStream = 32 | dataStream 33 | .process( 34 | new ProcessFunction() { 35 | @Override 36 | public void processElement( 37 | Row row, Context context, Collector collector) { 38 | collector.collect(row); 39 | } 40 | }) 41 | .returns(rowTypeInfo); 42 | 43 | Table inputTable = tableEnv.fromChangelogStream(processedStream); 44 | inputTable.printSchema(); 45 | 46 | inputTable.execute().print(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/conversion/WatermarkExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.conversion; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 5 | import org.apache.flink.table.api.Schema; 6 | import org.apache.flink.table.api.Table; 7 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 8 | import org.apache.flink.types.Row; 9 | 10 | public class WatermarkExample { 11 | 12 | public static void main(String[] args) throws Exception { 13 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 14 | StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); 15 | 16 | tableEnv.executeSql( 17 | "CREATE TABLE source (\n" + 18 | " id BIGINT,\n" + 19 | " price DECIMAL(32,2),\n" + 20 | " buyer STRING,\n" + 21 | " order_time TIMESTAMP(3),\n" + 22 | " WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND\n" + 23 | ") WITH (\n" + 24 | " 'connector' = 'datagen',\n" + 25 | " 'fields.id.min' = '1',\n" + 26 | " 'fields.id.max' = '1000',\n" + 27 | " 'fields.price.min' = '0',\n" + 28 | " 'fields.price.max' = '10000',\n" + 29 | " 'fields.buyer.length' = '5'\n" + 30 | ")"); 31 | Table table = tableEnv.sqlQuery("SELECT * FROM source"); 32 | DataStream dataStream = tableEnv.toDataStream(table); 33 | 34 | Table resultTable = tableEnv.fromDataStream(dataStream, 35 | Schema.newBuilder() 36 | .column("id", "BIGINT") 37 | .column("price", "DECIMAL(32,2)") 38 | .column("buyer", "STRING") 39 | .column("order_time", "TIMESTAMP(3)") 40 | .watermark("order_time", "SOURCE_WATERMARK()") 41 | .build()); 42 | resultTable.printSchema(); 43 | tableEnv.createTemporaryView("result_table", resultTable); 44 | 45 | Table table1 = tableEnv.sqlQuery( 46 | "SELECT window_start, window_end, SUM(price)\n" + 47 | "FROM TABLE(\n" + 48 | " TUMBLE(TABLE result_table, DESCRIPTOR(order_time), INTERVAL '10' SECONDS))\n" + 49 | "GROUP BY window_start, window_end"); 50 | table1.execute().print(); 51 | 52 | env.execute(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/cube/SimpleCubeExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.cube; 2 | 3 | import org.apache.flink.learning.udaf.GenericRecordAgg; 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 5 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 6 | 7 | import java.io.IOException; 8 | 9 | import static org.apache.flink.learning.table.utils.Utils.createTempFile; 10 | 11 | public class SimpleCubeExample { 12 | 13 | public static void main(String[] args) throws IOException { 14 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 15 | StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); 16 | 17 | tEnv.createTemporaryFunction("GenericRecordAgg", GenericRecordAgg.class); 18 | 19 | // user_id, product, count, price 20 | String contents = 21 | "1,A,1,10\n" + 22 | "2,A,3,27\n" + 23 | "3,B,4,30\n" + 24 | "4,B,5,70\n" + 25 | "5,A,1,10\n" + 26 | "6,C,2,30"; 27 | String path = createTempFile(contents); 28 | 29 | String source = 30 | "CREATE TABLE source (\n" 31 | + " user_id INT,\n" 32 | + " product STRING,\n" 33 | + " `count` STRING,\n" 34 | + " price STRING\n" 35 | + ") WITH (\n" 36 | + " 'connector.type' = 'filesystem',\n" 37 | + " 'connector.path' = '" 38 | + path 39 | + "',\n" 40 | + " 'format.type' = 'csv'\n" 41 | + ")"; 42 | tEnv.executeSql(source); 43 | 44 | String sql = 45 | "select\n" + 46 | "\tdims, GenericRecordAgg(metricData)\n" + 47 | "from (\n" + 48 | "\tselect \n" + 49 | "\t\tproduct as dims, \n" + 50 | "\t\tconcat('SUM:', `count`, '|', 'SUM:', price) as metricData\n" + 51 | "\tfrom source\n" + 52 | ") group by dims;"; 53 | 54 | tEnv.sqlQuery(sql).execute().print(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/sink/SinkModeTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.sink; 2 | 3 | import org.apache.flink.learning.table.utils.Order; 4 | import org.apache.flink.streaming.api.datastream.DataStream; 5 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 6 | import org.apache.flink.table.api.Table; 7 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 8 | 9 | import java.util.Arrays; 10 | 11 | public class SinkModeTest { 12 | 13 | public static void main(String[] args) throws Exception { 14 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 15 | env.setParallelism(1); 16 | StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); 17 | 18 | DataStream dataStream = 19 | env.fromCollection( 20 | Arrays.asList( 21 | new Order("1", "Bob", "Shoes", System.currentTimeMillis()), 22 | new Order("2", "Bob", "Shoes", System.currentTimeMillis()), 23 | new Order("3", "Julia", "Hat", System.currentTimeMillis()))); 24 | 25 | Table table = tEnv.fromDataStream(dataStream); 26 | 27 | Table result = tEnv.sqlQuery("SELECT count(*) from " + table); 28 | 29 | tEnv.toChangelogStream(result).print(); 30 | 31 | env.execute(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/utils/Order.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.utils; 2 | 3 | public class Order { 4 | 5 | private String orderId; 6 | private String userName; 7 | private String item; 8 | private long timestamp; 9 | 10 | public Order() { 11 | } 12 | 13 | public Order(String orderId, String userName, String item, long timestamp) { 14 | this.orderId = orderId; 15 | this.userName = userName; 16 | this.item = item; 17 | this.timestamp = timestamp; 18 | } 19 | 20 | public String getOrderId() { 21 | return orderId; 22 | } 23 | 24 | public void setOrderId(String orderId) { 25 | this.orderId = orderId; 26 | } 27 | 28 | public String getUserName() { 29 | return userName; 30 | } 31 | 32 | public void setUserName(String userName) { 33 | this.userName = userName; 34 | } 35 | 36 | public String getItem() { 37 | return item; 38 | } 39 | 40 | public void setItem(String item) { 41 | this.item = item; 42 | } 43 | 44 | public long getTimestamp() { 45 | return timestamp; 46 | } 47 | 48 | public void setTimestamp(long timestamp) { 49 | this.timestamp = timestamp; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/utils/OrderMapper.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.utils; 2 | 3 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.util.Collector; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.LocalDateTime; 10 | import java.time.ZoneOffset; 11 | import java.time.format.DateTimeFormatter; 12 | 13 | public class OrderMapper extends RichFlatMapFunction { 14 | 15 | private static Logger LOGGER = LoggerFactory.getLogger(OrderMapper.class); 16 | 17 | private DateTimeFormatter formatter; 18 | 19 | @Override 20 | public void open(Configuration parameters) throws Exception { 21 | this.formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 22 | } 23 | 24 | @Override 25 | public void flatMap(String line, Collector collector) throws Exception { 26 | try { 27 | String[] items = line.split(","); 28 | String orderId = items[0]; 29 | String userName = items[1]; 30 | String item = items[2]; 31 | long timestamp = LocalDateTime.parse(items[3], formatter).toInstant(ZoneOffset.of("+8")).toEpochMilli(); 32 | collector.collect(new Order(orderId, userName, item, timestamp)); 33 | } catch (Exception e) { 34 | LOGGER.error("Cannot parse line: {}", line); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/utils/Shipment.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.utils; 2 | 3 | public class Shipment { 4 | 5 | private String shipId; 6 | private String orderId; 7 | private String company; 8 | private long timestamp; 9 | 10 | public Shipment(String shipId, String orderId, String company, long timestamp) { 11 | this.shipId = shipId; 12 | this.orderId = orderId; 13 | this.company = company; 14 | this.timestamp = timestamp; 15 | } 16 | 17 | public String getShipId() { 18 | return shipId; 19 | } 20 | 21 | public void setShipId(String shipId) { 22 | this.shipId = shipId; 23 | } 24 | 25 | public String getOrderId() { 26 | return orderId; 27 | } 28 | 29 | public void setOrderId(String orderId) { 30 | this.orderId = orderId; 31 | } 32 | 33 | public String getCompany() { 34 | return company; 35 | } 36 | 37 | public void setCompany(String company) { 38 | this.company = company; 39 | } 40 | 41 | public long getTimestamp() { 42 | return timestamp; 43 | } 44 | 45 | public void setTimestamp(long timestamp) { 46 | this.timestamp = timestamp; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/utils/ShipmentMapper.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.table.utils; 2 | 3 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.util.Collector; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.LocalDateTime; 10 | import java.time.ZoneOffset; 11 | import java.time.format.DateTimeFormatter; 12 | 13 | public class ShipmentMapper extends RichFlatMapFunction { 14 | 15 | private static Logger LOGGER = LoggerFactory.getLogger(ShipmentMapper.class); 16 | 17 | private DateTimeFormatter formatter; 18 | 19 | @Override 20 | public void open(Configuration parameters) throws Exception { 21 | this.formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 22 | } 23 | 24 | @Override 25 | public void flatMap(String line, Collector collector) throws Exception { 26 | try { 27 | String[] items = line.split(","); 28 | String shipId = items[0]; 29 | String orderId = items[1]; 30 | String company = items[2]; 31 | long timestamp = LocalDateTime.parse(items[3], formatter).toInstant(ZoneOffset.of("+8")).toEpochMilli(); 32 | collector.collect(new Shipment(shipId, orderId, company, timestamp)); 33 | } catch (Exception e) { 34 | LOGGER.error("Cannot parse line: {}", line); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/java/org/apache/flink/learning/table/utils/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.flink.learning.table.utils; 19 | 20 | import org.apache.flink.util.FileUtils; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.util.Scanner; 26 | import java.util.UUID; 27 | 28 | /** Common utils of examples. */ 29 | public class Utils { 30 | 31 | /** Creates a temporary file with the contents and returns the absolute path. */ 32 | public static String createTempFile(String contents) throws IOException { 33 | File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".csv"); 34 | tempFile.deleteOnExit(); 35 | FileUtils.writeFileUtf8(tempFile, contents); 36 | return tempFile.toURI().toString(); 37 | } 38 | 39 | public static String readResourceFile(String filePath, ClassLoader classLoader) 40 | throws IOException { 41 | InputStream inputStream = classLoader.getResourceAsStream(filePath); 42 | if (inputStream == null) { 43 | throw new IOException("No file names " + filePath); 44 | } 45 | 46 | try (Scanner scanner = new Scanner(inputStream, "UTF-8")) { 47 | StringBuilder content = new StringBuilder(); 48 | while (scanner.hasNextLine()) { 49 | content.append(scanner.nextLine()); 50 | content.append(System.lineSeparator()); 51 | } 52 | 53 | return content.toString(); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /flink-learning/flink-sql/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | rootLogger.level = INFO 2 | rootLogger.appenderRef.console.ref = ConsoleAppender 3 | 4 | appender.console.name = ConsoleAppender 5 | appender.console.type = CONSOLE 6 | appender.console.layout.type = PatternLayout 7 | appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n 8 | -------------------------------------------------------------------------------- /flink-learning/flink-udx/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.apache.flink.learning 8 | flink-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | flink-udx 13 | 14 | 15 | 16 | org.apache.flink 17 | flink-table-common 18 | 19 | 20 | -------------------------------------------------------------------------------- /flink-learning/flink-udx/src/main/java/org/apache/flink/learning/udaf/GenericRecord.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.udaf; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | 7 | public class GenericRecord { 8 | 9 | private final List aggType; 10 | private final List values; 11 | 12 | public GenericRecord(String metricsData) { 13 | this.aggType = new ArrayList<>(); 14 | this.values = new ArrayList<>(); 15 | 16 | String[] items = metricsData.split("\\|"); 17 | for (String s : items) { 18 | String[] item = s.split(":"); 19 | aggType.add(item[0]); 20 | values.add(Integer.parseInt(item[1])); 21 | } 22 | } 23 | 24 | public String getMetricsData() { 25 | return values.stream().map(String::valueOf).collect(Collectors.joining(":")); 26 | } 27 | 28 | public String getAggType(int i) { 29 | return aggType.get(i); 30 | } 31 | 32 | public Integer getValue(int i) { 33 | return values.get(i); 34 | } 35 | 36 | public int getSize() { 37 | return aggType.size(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /flink-learning/flink-udx/src/main/java/org/apache/flink/learning/udaf/GenericRecordAccumulator.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.udaf; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | 7 | public class GenericRecordAccumulator { 8 | 9 | public List aggValues = new ArrayList<>(); 10 | 11 | public void acc(GenericRecord record) { 12 | if (aggValues.isEmpty()) { 13 | aggValues = new ArrayList<>(); 14 | for (int i = 0; i < record.getSize(); ++i) { 15 | aggValues.add(0); 16 | } 17 | } 18 | 19 | for (int i = 0; i < record.getSize(); i++) { 20 | switch (record.getAggType(i)) { 21 | case "SUM": 22 | aggValues.set(i, aggValues.get(i) + record.getValue(i)); 23 | break; 24 | case "COUNT": 25 | aggValues.set(i, aggValues.get(i) + 1); 26 | break; 27 | default: 28 | break; 29 | } 30 | } 31 | } 32 | 33 | public String getAggValues() { 34 | return aggValues.stream().map(String::valueOf).collect(Collectors.joining(":")); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /flink-learning/flink-udx/src/main/java/org/apache/flink/learning/udaf/GenericRecordAgg.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.learning.udaf; 2 | 3 | import org.apache.flink.table.annotation.DataTypeHint; 4 | import org.apache.flink.table.functions.AggregateFunction; 5 | 6 | 7 | public class GenericRecordAgg 8 | extends AggregateFunction { 9 | 10 | @Override 11 | public String getValue(GenericRecordAccumulator acc) { 12 | return acc.getAggValues(); 13 | } 14 | 15 | @Override 16 | public GenericRecordAccumulator createAccumulator() { 17 | return new GenericRecordAccumulator(); 18 | } 19 | 20 | public void accumulate(GenericRecordAccumulator acc, String metricData) { 21 | GenericRecord record = new GenericRecord(metricData); 22 | acc.acc(record); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /hadoop-learning/hdfs-learning/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hadoop-learning 5 | org.example 6 | 1.0-SNAPSHOT 7 | 8 | 4.0.0 9 | hdfs-learning 10 | 11 | 12 | org.slf4j 13 | slf4j-log4j12 14 | 1.7.30 15 | compile 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /hadoop-learning/hdfs-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hadoop-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | hdfs-learning 13 | 14 | 15 | 16 | org.apache.hadoop 17 | hadoop-client 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /hadoop-learning/hdfs-learning/src/main/java/HDFSClientExample.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.conf.Configuration; 2 | import org.apache.hadoop.fs.*; 3 | import org.junit.Test; 4 | 5 | import java.io.IOException; 6 | import java.net.URI; 7 | import java.net.URISyntaxException; 8 | import java.nio.ByteBuffer; 9 | 10 | public class HDFSClientExample { 11 | 12 | public static String HDFS_URL = "hdfs://node1:8020"; 13 | 14 | @Test 15 | public void testListFiles() throws URISyntaxException, IOException, InterruptedException { 16 | Configuration configuration = new Configuration(); 17 | // FileSystem fs = FileSystem.get(new URI("hdfs://hadoop102:8020"), configuration); 18 | FileSystem fs = FileSystem.get(new URI(HDFS_URL), configuration); 19 | RemoteIterator it = fs.listFiles(new Path("/"), false); 20 | while (it.hasNext()) { 21 | System.out.println(it.next()); 22 | } 23 | } 24 | 25 | @Test 26 | public void testMkdirs() throws IOException, URISyntaxException, InterruptedException { 27 | Configuration configuration = new Configuration(); 28 | // FileSystem fs = FileSystem.get(new URI("hdfs://hadoop102:8020"), configuration); 29 | FileSystem fs = FileSystem.get(new URI(HDFS_URL), configuration, "admin"); 30 | fs.mkdirs(new Path("/xiyou/huaguoshan/")); 31 | fs.close(); 32 | } 33 | 34 | @Test 35 | public void testWriteData() throws IOException, URISyntaxException, InterruptedException { 36 | Configuration conf = new Configuration(); 37 | FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "liebing"); 38 | Path path = new Path("/test.txt"); 39 | FSDataOutputStream out = fs.create(path); 40 | out.writeBytes("test-data"); 41 | out.close(); 42 | fs.close(); 43 | } 44 | 45 | @Test 46 | public void testReadData() throws URISyntaxException, IOException, InterruptedException { 47 | Configuration conf = new Configuration(); 48 | FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "liebing"); 49 | Path path = new Path("/test.txt"); 50 | FSDataInputStream in = fs.open(path); 51 | ByteBuffer bb = ByteBuffer.allocate(1024); 52 | in.read(2, bb); 53 | System.out.println(new String(bb.array())); 54 | in.close(); 55 | fs.close(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /hadoop-learning/hdfs-learning/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.encoding=UTF-8 5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.console.layout.ConversionPattern=[%p]\t%d\t[%t]\t%c{3}\t(%F:%L)\t-%m%n -------------------------------------------------------------------------------- /hadoop-learning/mapreduce-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hadoop-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | mapreduce-learning 13 | 14 | 15 | 16 | org.apache.hadoop 17 | hadoop-client 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /hadoop-learning/mapreduce-learning/src/main/java/WordCount.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import java.util.StringTokenizer; 3 | 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.IntWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 13 | import org.apache.hadoop.util.GenericOptionsParser; 14 | 15 | public class WordCount { 16 | 17 | public static class TokenizerMapper 18 | extends Mapper{ 19 | 20 | private final static IntWritable one = new IntWritable(1); 21 | private Text word = new Text(); 22 | 23 | public void map(Object key, Text value, Context context) throws IOException, InterruptedException { 24 | StringTokenizer itr = new StringTokenizer(value.toString()); 25 | while (itr.hasMoreTokens()) { 26 | word.set(itr.nextToken()); 27 | context.write(word, one); 28 | } 29 | } 30 | } 31 | 32 | public static class IntSumReducer 33 | extends Reducer { 34 | private IntWritable result = new IntWritable(); 35 | 36 | public void reduce(Text key, Iterable values, 37 | Context context 38 | ) throws IOException, InterruptedException { 39 | int sum = 0; 40 | for (IntWritable val : values) { 41 | sum += val.get(); 42 | } 43 | result.set(sum); 44 | context.write(key, result); 45 | } 46 | } 47 | 48 | public static void main(String[] args) throws Exception { 49 | Configuration conf = new Configuration(); 50 | String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); 51 | if (otherArgs.length < 2) { 52 | System.err.println("Usage: wordcount [...] "); 53 | System.exit(2); 54 | } 55 | Job job = Job.getInstance(conf, "word count"); 56 | job.setJarByClass(WordCount.class); 57 | job.setMapperClass(TokenizerMapper.class); 58 | job.setCombinerClass(IntSumReducer.class); 59 | job.setReducerClass(IntSumReducer.class); 60 | job.setOutputKeyClass(Text.class); 61 | job.setOutputValueClass(IntWritable.class); 62 | for (int i = 0; i < otherArgs.length - 1; ++i) { 63 | FileInputFormat.addInputPath(job, new Path(otherArgs[i])); 64 | } 65 | FileOutputFormat.setOutputPath(job, 66 | new Path(otherArgs[otherArgs.length - 1])); 67 | System.exit(job.waitForCompletion(true) ? 0 : 1); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /hadoop-learning/mapreduce-learning/src/main/resources/HTTP_20130313143750.dat: -------------------------------------------------------------------------------- 1 | 1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200 2 | 1363157995052 13826544101 5C-0E-8B-C7-F1-E0:CMCC 120.197.40.4 4 0 264 0 200 3 | 1363157991076 13926435656 20-10-7A-28-CC-0A:CMCC 120.196.100.99 2 4 132 1512 200 4 | 1363154400022 13926251106 5C-0E-8B-8B-B1-50:CMCC 120.197.40.4 4 0 240 0 200 5 | 1363157993044 18211575961 94-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 iface.qiyi.com 视频网站 15 12 1527 2106 200 6 | 1363157995074 84138413 5C-0E-8B-8C-E8-20:7DaysInn 120.197.40.4 122.72.52.12 20 16 4116 1432 200 7 | 1363157993055 13560439658 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 18 15 1116 954 200 8 | 1363157995033 15920133257 5C-0E-8B-C7-BA-20:CMCC 120.197.40.4 sug.so.360.cn 信息安全 20 20 3156 2936 200 9 | 1363157983019 13719199419 68-A1-B7-03-07-B1:CMCC-EASY 120.196.100.82 4 0 240 0 200 10 | 1363157984041 13660577991 5C-0E-8B-92-5C-20:CMCC-EASY 120.197.40.4 s19.cnzz.com 站点统计 24 9 6960 690 200 11 | 1363157973098 15013685858 5C-0E-8B-C7-F7-90:CMCC 120.197.40.4 rank.ie.sogou.com 搜索引擎 28 27 3659 3538 200 12 | 1363157986029 15989002119 E8-99-C4-4E-93-E0:CMCC-EASY 120.196.100.99 www.umeng.com 站点统计 3 3 1938 180 200 13 | 1363157992093 13560439658 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 15 9 918 4938 200 14 | 1363157986041 13480253104 5C-0E-8B-C7-FC-80:CMCC-EASY 120.197.40.4 3 3 180 180 200 15 | 1363157984040 13602846565 5C-0E-8B-8B-B6-00:CMCC 120.197.40.4 2052.flash2-http.qq.com 综合门户 15 12 1938 2910 200 16 | 1363157995093 13922314466 00-FD-07-A2-EC-BA:CMCC 120.196.100.82 img.qfc.cn 12 12 3008 3720 200 17 | 1363157982040 13502468823 5C-0A-5B-6A-0B-D4:CMCC-EASY 120.196.100.99 y0.ifengimg.com 综合门户 57 102 7335 110349 200 18 | 1363157986072 18320173382 84-25-DB-4F-10-1A:CMCC-EASY 120.196.100.99 input.shouji.sogou.com 搜索引擎 21 18 9531 2412 200 19 | 1363157990043 13925057413 00-1F-64-E1-E6-9A:CMCC 120.196.100.55 t3.baidu.com 搜索引擎 69 63 11058 48243 200 20 | 1363157988072 13760778710 00-FD-07-A4-7B-08:CMCC 120.196.100.82 2 2 120 120 200 21 | 1363157985066 13726238888 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200 22 | 1363157993055 13560436666 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 18 15 1116 954 200 -------------------------------------------------------------------------------- /hadoop-learning/mapreduce-learning/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.encoding=UTF-8 5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.console.layout.ConversionPattern=[%p]\t%d\t[%t]\t%c{3}\t(%F:%L)\t-%m%n -------------------------------------------------------------------------------- /hadoop-learning/mapreduce-learning/src/main/resources/wordcount.txt: -------------------------------------------------------------------------------- 1 | How are you 2 | I am Jack 3 | What is your name 4 | Where are you from -------------------------------------------------------------------------------- /hadoop-learning/rpc-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hadoop-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | rpc-learning 13 | 14 | 15 | 16 | org.apache.hadoop 17 | hadoop-common 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /hadoop-learning/rpc-learning/src/main/java/Client.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.conf.Configuration; 2 | import org.apache.hadoop.ipc.RPC; 3 | 4 | import java.io.IOException; 5 | import java.net.InetSocketAddress; 6 | 7 | public class Client { 8 | 9 | public static void main(String[] args) { 10 | try { 11 | MyInterface proxy = RPC.getProxy( 12 | MyInterface.class, 13 | 1L, 14 | new InetSocketAddress("localhost", 12345), 15 | new Configuration()); 16 | int res = proxy.add(1, 2); 17 | System.out.println(res); 18 | } catch (IOException e) { 19 | e.printStackTrace(); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hadoop-learning/rpc-learning/src/main/java/MyInterface.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.ipc.VersionedProtocol; 2 | 3 | public interface MyInterface extends VersionedProtocol { 4 | long versionID = 1L; 5 | 6 | int add(int num1, int num2); 7 | } 8 | -------------------------------------------------------------------------------- /hadoop-learning/rpc-learning/src/main/java/MyInterfaceImpl.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.ipc.ProtocolSignature; 2 | 3 | import java.io.IOException; 4 | 5 | public class MyInterfaceImpl implements MyInterface { 6 | 7 | @Override 8 | public int add(int num1, int num2) { 9 | return num1 + num2; 10 | } 11 | 12 | @Override 13 | public long getProtocolVersion(String s, long l) throws IOException { 14 | return MyInterface.versionID; 15 | } 16 | 17 | @Override 18 | public ProtocolSignature getProtocolSignature(String s, long l, int i) throws IOException { 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /hadoop-learning/rpc-learning/src/main/java/Server.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.conf.Configuration; 2 | import org.apache.hadoop.ipc.RPC; 3 | 4 | import java.io.IOException; 5 | 6 | public class Server { 7 | 8 | public static void main(String[] args) { 9 | RPC.Builder builder = new RPC.Builder(new Configuration()); 10 | builder.setBindAddress("localhost"); 11 | builder.setPort(12345); 12 | builder.setProtocol(MyInterface.class); 13 | builder.setInstance(new MyInterfaceImpl()); 14 | 15 | try { 16 | RPC.Server server = builder.build(); 17 | server.start(); 18 | } catch (IOException e) { 19 | e.printStackTrace(); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hadoop-learning 5 | org.example 6 | 1.0-SNAPSHOT 7 | 8 | 4.0.0 9 | yarn-learning 10 | 11 | 12 | org.slf4j 13 | slf4j-log4j12 14 | 1.7.30 15 | compile 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hadoop-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | yarn-learning 13 | 14 | 15 | 16 | org.apache.hadoop 17 | hadoop-hdfs 18 | ${hadoop.version} 19 | 20 | 21 | org.apache.hadoop 22 | hadoop-client 23 | ${hadoop.version} 24 | 25 | 26 | org.apache.hadoop 27 | hadoop-yarn-api 28 | ${hadoop.version} 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.yarn.applications.distributedshell; 2 | 3 | import org.apache.hadoop.classification.InterfaceAudience; 4 | import org.apache.hadoop.classification.InterfaceStability; 5 | 6 | /** 7 | * Constants used in both Client and Application Master 8 | */ 9 | @InterfaceAudience.Public 10 | @InterfaceStability.Unstable 11 | public class DSConstants { 12 | 13 | /** 14 | * Environment key name pointing to the shell script's location 15 | */ 16 | public static final String DISTRIBUTEDSHELLSCRIPTLOCATION = "DISTRIBUTEDSHELLSCRIPTLOCATION"; 17 | 18 | /** 19 | * Environment key name denoting the file timestamp for the shell script. 20 | * Used to validate the local resource. 21 | */ 22 | public static final String DISTRIBUTEDSHELLSCRIPTTIMESTAMP = "DISTRIBUTEDSHELLSCRIPTTIMESTAMP"; 23 | 24 | /** 25 | * Environment key name denoting the file content length for the shell script. 26 | * Used to validate the local resource. 27 | */ 28 | public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN"; 29 | 30 | /** 31 | * Environment key name denoting the timeline domain ID. 32 | */ 33 | public static final String DISTRIBUTEDSHELLTIMELINEDOMAIN = "DISTRIBUTEDSHELLTIMELINEDOMAIN"; 34 | } 35 | -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.yarn.applications.distributedshell; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.InputStream; 5 | import java.util.Map.Entry; 6 | import java.util.Properties; 7 | 8 | import org.apache.commons.io.IOUtils; 9 | import org.apache.log4j.LogManager; 10 | import org.apache.log4j.PropertyConfigurator; 11 | 12 | /** 13 | * Helper class to over write log4j configuration. 14 | * */ 15 | public class Log4jPropertyHelper { 16 | 17 | /** 18 | * Over write the targetClass's log4j configuration with custom configuration in log4jPath. 19 | * */ 20 | public static void updateLog4jConfiguration(Class targetClass, 21 | String log4jPath) throws Exception { 22 | Properties customProperties = new Properties(); 23 | FileInputStream fs = null; 24 | InputStream is = null; 25 | try { 26 | fs = new FileInputStream(log4jPath); 27 | is = targetClass.getResourceAsStream("/log4j.properties"); 28 | customProperties.load(fs); 29 | Properties originalProperties = new Properties(); 30 | originalProperties.load(is); 31 | for (Entry entry : customProperties.entrySet()) { 32 | originalProperties.setProperty(entry.getKey().toString(), entry 33 | .getValue().toString()); 34 | } 35 | LogManager.resetConfiguration(); 36 | PropertyConfigurator.configure(originalProperties); 37 | } finally { 38 | IOUtils.closeQuietly(is); 39 | IOUtils.closeQuietly(fs); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/readme.md: -------------------------------------------------------------------------------- 1 | This simple example shows the way to implement new Applications for YARN. It will run the Shell command or script 2 | provided by the user in the YARN Container 3 | 4 | To run the application you need to compile it first. 5 | ```shell 6 | cd hadoop-learning 7 | mvn clean package -DskipTests 8 | cd yarn-learning/target/ 9 | ``` 10 | 11 | Then you can run it by the following command. 12 | ```shell 13 | yarn jar yarn-learning-1.0-SNAPSHOT.jar \ 14 | org.apache.hadoop.yarn.applications.distributedshell.Client \ 15 | -jar yarn-learning-1.0-SNAPSHOT.jar \ 16 | -shell_script shell.sh \ 17 | -num_containers 3 18 | ``` -------------------------------------------------------------------------------- /hadoop-learning/yarn-learning/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.encoding=UTF-8 5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.console.layout.ConversionPattern=[%p]\t%d\t[%t]\t%c{3}\t(%F:%L)\t-%m%n -------------------------------------------------------------------------------- /hbase-learning/hbase1/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hbase-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | hbase1 13 | 14 | 15 | 1.4.14 16 | 17 | 18 | 19 | 20 | org.apache.hbase 21 | hbase-client 22 | ${hbase.version} 23 | 24 | 25 | netty-all 26 | io.netty 27 | 28 | 29 | 30 | 31 | org.apache.hbase 32 | hbase-server 33 | ${hbase.version} 34 | 35 | 36 | netty-all 37 | io.netty 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/java/org/apache/hbase/learning/filter/FilterExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.learning.filter; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.Cell; 5 | import org.apache.hadoop.hbase.HBaseConfiguration; 6 | import org.apache.hadoop.hbase.TableName; 7 | import org.apache.hadoop.hbase.client.*; 8 | 9 | import java.io.IOException; 10 | 11 | import static org.apache.hbase.learning.filter.Util.cellToString; 12 | 13 | public class FilterExample { 14 | 15 | public static void main(String[] args) throws IOException { 16 | Configuration conf = HBaseConfiguration.create(); 17 | Connection connection = ConnectionFactory.createConnection(conf); 18 | Table table = connection.getTable(TableName.valueOf("filter_test")); 19 | Scan scan = new Scan(); 20 | scan.setFilter(new CustomFilter("1506".getBytes(), "f2".getBytes(), "d".getBytes(), "12".getBytes())); 21 | ResultScanner scanner = table.getScanner(scan); 22 | for (Result result : scanner) { 23 | for (Cell cell : result.rawCells()) { 24 | System.out.println(cellToString(cell)); 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/java/org/apache/hbase/learning/filter/Test.java: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.learning.filter; 2 | 3 | import java.io.File; 4 | 5 | public class Test { 6 | 7 | public static void main(String[] args) { 8 | File file = new File("/home/liebing/Code/data_systems_learning/hbase-learning/hbase1/src/main/java/org/apache/hbase/learning/filter/FilterExample.java"); 9 | System.out.println(file.getParent()); 10 | System.out.println(file.getName()); 11 | 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/java/org/apache/hbase/learning/filter/TestRowMutations.java: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.learning.filter; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.HBaseConfiguration; 5 | import org.apache.hadoop.hbase.TableName; 6 | import org.apache.hadoop.hbase.client.*; 7 | import org.apache.hadoop.hbase.util.Bytes; 8 | 9 | import java.io.IOException; 10 | 11 | public class TestRowMutations { 12 | 13 | public static void main(String[] args) throws IOException { 14 | Configuration conf = HBaseConfiguration.create(); 15 | Connection connection = ConnectionFactory.createConnection(conf); 16 | BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("test")); 17 | 18 | byte[] rowKey = Bytes.toBytes("row1"); 19 | byte[] cf = Bytes.toBytes("cf"); 20 | byte[] q1 = Bytes.toBytes("q1"); 21 | byte[] q2 = Bytes.toBytes("q2"); 22 | byte[] v1 = Bytes.toBytes("v1"); 23 | byte[] v2 = Bytes.toBytes("v2"); 24 | 25 | Put put = new Put(rowKey); 26 | put.addColumn(cf, q1, v1); 27 | put.addColumn(cf, q2, v2); 28 | 29 | Delete delete = new Delete(rowKey); 30 | delete.addColumn(cf, q2); 31 | 32 | mutator.mutate(put); 33 | mutator.mutate(delete); 34 | 35 | mutator.flush(); 36 | mutator.close(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/java/org/apache/hbase/learning/filter/Util.java: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.learning.filter; 2 | 3 | import org.apache.hadoop.hbase.Cell; 4 | import org.apache.hadoop.hbase.CellUtil; 5 | 6 | public class Util { 7 | 8 | public static String cellToString(Cell cell) { 9 | String row = new String(CellUtil.cloneRow(cell)); 10 | String family = new String(CellUtil.cloneFamily(cell)); 11 | String qualify = new String(CellUtil.cloneQualifier(cell)); 12 | String value = new String(CellUtil.cloneValue(cell)); 13 | return String.format("row key: %s, family: %s, qualify: %s, value: %s", 14 | row, family, qualify, value); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 23 | 24 | 25 | hbase.cluster.distributed 26 | true 27 | 28 | 29 | hbase.rootdir 30 | hdfs://localhost:9000/hbase 31 | 32 | 33 | 34 | hbase.zookeeper.property.dataDir 35 | /usr/local/hbase-1.4.14/data/zookeeper 36 | 37 | 38 | zookeeper.znode.parent 39 | /hbase 40 | 41 | 42 | hbase.zookeeper.property.clientPort 43 | 2181 44 | 45 | 46 | hbase.zookeeper.quorum 47 | localhost 48 | 49 | 50 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/scala/org/apache/hbase/bulkload/BulkLoadPartitioner.scala: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.bulkload 2 | 3 | import java.util 4 | import java.util.Comparator 5 | 6 | import org.apache.yetus.audience.InterfaceAudience; 7 | import org.apache.hadoop.hbase.util.Bytes 8 | import org.apache.spark.Partitioner 9 | 10 | /** 11 | * A Partitioner implementation that will separate records to different 12 | * HBase Regions based on region splits 13 | * 14 | * @param startKeys The start keys for the given table 15 | */ 16 | @InterfaceAudience.Public 17 | class BulkLoadPartitioner(startKeys:Array[Array[Byte]]) 18 | extends Partitioner { 19 | // when table not exist, startKeys = Byte[0][] 20 | override def numPartitions: Int = if (startKeys.length == 0) 1 else startKeys.length 21 | 22 | override def getPartition(key: Any): Int = { 23 | 24 | val comparator: Comparator[Array[Byte]] = (o1: Array[Byte], o2: Array[Byte]) => { 25 | Bytes.compareTo(o1, o2) 26 | } 27 | 28 | val rowKey:Array[Byte] = 29 | key match { 30 | case qualifier: KeyFamilyQualifier => 31 | qualifier.rowKey 32 | case wrapper: ByteArrayWrapper => 33 | wrapper.value 34 | case _ => 35 | key.asInstanceOf[Array[Byte]] 36 | } 37 | var partition = util.Arrays.binarySearch(startKeys, rowKey, comparator) 38 | if (partition < 0) 39 | partition = partition * -1 + -2 40 | if (partition < 0) 41 | partition = 0 42 | partition 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/scala/org/apache/hbase/bulkload/ByteArrayWrapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.bulkload 2 | 3 | import java.io.Serializable 4 | 5 | import org.apache.yetus.audience.InterfaceAudience; 6 | import org.apache.hadoop.hbase.util.Bytes 7 | 8 | /** 9 | * This is a wrapper over a byte array so it can work as 10 | * a key in a hashMap 11 | * 12 | * @param value The Byte Array value 13 | */ 14 | @InterfaceAudience.Public 15 | class ByteArrayWrapper (var value:Array[Byte]) 16 | extends Comparable[ByteArrayWrapper] with Serializable { 17 | override def compareTo(valueOther: ByteArrayWrapper): Int = { 18 | Bytes.compareTo(value,valueOther.value) 19 | } 20 | override def equals(o2: Any): Boolean = { 21 | o2 match { 22 | case wrapper: ByteArrayWrapper => 23 | Bytes.equals(value, wrapper.value) 24 | case _ => 25 | false 26 | } 27 | } 28 | override def hashCode():Int = { 29 | Bytes.hashCode(value) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/scala/org/apache/hbase/bulkload/KeyFamilyQualifier.scala: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.bulkload 2 | 3 | import java.io.Serializable 4 | 5 | import org.apache.yetus.audience.InterfaceAudience; 6 | import org.apache.hadoop.hbase.util.Bytes 7 | 8 | /** 9 | * This is the key to be used for sorting and shuffling. 10 | * 11 | * We will only partition on the rowKey but we will sort on all three 12 | * 13 | * @param rowKey Record RowKey 14 | * @param family Record ColumnFamily 15 | * @param qualifier Cell Qualifier 16 | */ 17 | @InterfaceAudience.Public 18 | class KeyFamilyQualifier(val rowKey:Array[Byte], val family:Array[Byte], val qualifier:Array[Byte]) 19 | extends Comparable[KeyFamilyQualifier] with Serializable { 20 | override def compareTo(o: KeyFamilyQualifier): Int = { 21 | var result = Bytes.compareTo(rowKey, o.rowKey) 22 | if (result == 0) { 23 | result = Bytes.compareTo(family, o.family) 24 | if (result == 0) result = Bytes.compareTo(qualifier, o.qualifier) 25 | } 26 | result 27 | } 28 | override def toString: String = { 29 | Bytes.toString(rowKey) + ":" + Bytes.toString(family) + ":" + Bytes.toString(qualifier) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hbase-learning/hbase1/src/main/scala/org/apache/hbase/learning/HBaseSparkBulkLoad.scala: -------------------------------------------------------------------------------- 1 | package org.apache.hbase.learning 2 | 3 | import org.apache.hadoop.fs.Path 4 | import org.apache.hadoop.hbase.client.ConnectionFactory 5 | import org.apache.hadoop.hbase.io.ImmutableBytesWritable 6 | import org.apache.hadoop.hbase.mapreduce.{HFileOutputFormat2, LoadIncrementalHFiles} 7 | import org.apache.hadoop.hbase.util.Bytes 8 | import org.apache.hadoop.hbase.{HBaseConfiguration, KeyValue, TableName} 9 | import org.apache.hbase.bulkload.{BulkLoadPartitioner, KeyFamilyQualifier} 10 | import org.apache.spark.{SparkConf, SparkContext} 11 | 12 | /** 13 | * A simple example shows how to use spark to perform bulk load of hbase. 14 | * */ 15 | object HBaseSparkBulkLoad { 16 | 17 | def main(args: Array[String]): Unit = { 18 | if (args.length < 2) { 19 | println("Usage: HBaseSparkBulkLoad ") 20 | System.exit(1) 21 | } 22 | val tableName = args(0) 23 | val columnFamily = args(1) 24 | val stagingFolder = args(2) 25 | 26 | // get table info 27 | val hConf = HBaseConfiguration.create() 28 | val connection = ConnectionFactory.createConnection(hConf) 29 | val table = connection.getTable(TableName.valueOf(tableName)) 30 | val regionLocator = connection.getRegionLocator(TableName.valueOf(tableName)) 31 | 32 | val sConf = new SparkConf().setAppName("bulkload") 33 | sConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") 34 | val sc = new SparkContext(sConf) 35 | 36 | // simple data to bulk load to hbase 37 | val rdd = sc.parallelize(Array( 38 | (Bytes.toBytes("4"), Bytes.toBytes(columnFamily), Bytes.toBytes("4"), Bytes.toBytes("1")), 39 | (Bytes.toBytes("1"), Bytes.toBytes(columnFamily), Bytes.toBytes("2"), Bytes.toBytes("2")) 40 | )).repartition(2) // random partition to simulate a real scene 41 | 42 | // parse the source data and repartition and sort by row key, family, qualify 43 | val sortedRdd = rdd.map(line => { 44 | val keyFamilyQualifier = new KeyFamilyQualifier(line._1, line._2, line._3) 45 | val value = line._4 46 | (keyFamilyQualifier, value) 47 | }).repartitionAndSortWithinPartitions(new BulkLoadPartitioner(regionLocator.getStartKeys)) 48 | // reformat the data so that we can save as HFileOutputFormat2 49 | val hfileRdd = sortedRdd.map(line => { 50 | val rowKey = new ImmutableBytesWritable(line._1.rowKey) 51 | val keyValue = new KeyValue(line._1.rowKey, line._1.family, line._1.qualifier, line._2) 52 | (rowKey, keyValue) 53 | }) 54 | 55 | // save the rdd as hfile 56 | hfileRdd.saveAsNewAPIHadoopFile( 57 | stagingFolder, 58 | classOf[ImmutableBytesWritable], 59 | classOf[KeyValue], 60 | classOf[HFileOutputFormat2], hConf) 61 | // load the hfile from hdfs 62 | val loader = new LoadIncrementalHFiles(hConf) 63 | loader.doBulkLoad(new Path(stagingFolder), connection.getAdmin, table, regionLocator) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /hbase-learning/hbase2/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | hbase-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | hbase2 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | -------------------------------------------------------------------------------- /hbase-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | hbase-learning 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | hbase1 13 | hbase2 14 | 15 | 16 | 17 | 1.8 18 | 3.2.1 19 | 20 | 21 | 22 | 23 | org.apache.spark 24 | spark-core_2.12 25 | ${spark.version} 26 | provided 27 | 28 | 29 | org.apache.spark 30 | spark-sql_2.12 31 | ${spark.version} 32 | provided 33 | 34 | 35 | io.netty 36 | netty-all 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | org.apache.maven.plugins 46 | maven-compiler-plugin 47 | 3.8.0 48 | 49 | ${target.java.version} 50 | ${target.java.version} 51 | 52 | false 53 | 54 | 55 | -Xpkginfo:always 56 | 57 | 58 | 59 | 60 | 61 | net.alchim31.maven 62 | scala-maven-plugin 63 | 3.2.2 64 | 65 | 66 | -nobootcp 67 | -target:jvm-${target.java.version} 68 | 69 | 70 | -Xss2m 71 | 72 | 73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /hive-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | hive-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.3.9 13 | 2.7.7 14 | 15 | 16 | 17 | 18 | org.apache.hive 19 | hive-exec 20 | ${hive.version} 21 | 22 | 23 | org.apache.hadoop 24 | hadoop-client 25 | ${hadoop.version} 26 | provided 27 | 28 | 29 | 30 | 31 | 32 | 33 | org.apache.maven.plugins 34 | maven-compiler-plugin 35 | 36 | 8 37 | 8 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /hive-learning/src/main/java/MyStringLength.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.hive.ql.exec.UDFArgumentException; 2 | import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; 3 | import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; 4 | import org.apache.hadoop.hive.ql.metadata.HiveException; 5 | import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; 6 | import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; 7 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; 8 | 9 | public class MyStringLength extends GenericUDF { 10 | 11 | public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { 12 | // check arguments length 13 | if (args.length != 1) { 14 | throw new UDFArgumentLengthException("Input arguments length error"); 15 | } 16 | // check arguments type 17 | if (!args[0].getCategory().equals(ObjectInspector.Category.PRIMITIVE)) { 18 | throw new UDFArgumentTypeException(0, "Input arguments type error"); 19 | } 20 | return PrimitiveObjectInspectorFactory.javaIntObjectInspector; 21 | } 22 | 23 | public Object evaluate(DeferredObject[] args) throws HiveException { 24 | if (args[0].get() == null) { 25 | return 0; 26 | } 27 | return args[0].get().toString().length(); 28 | } 29 | 30 | public String getDisplayString(String[] strings) { 31 | return ""; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /hive-learning/src/main/java/MyUDTF.java: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.hive.ql.exec.UDFArgumentException; 2 | import org.apache.hadoop.hive.ql.metadata.HiveException; 3 | import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; 4 | import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; 5 | import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; 6 | import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; 7 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | public class MyUDTF extends GenericUDTF { 13 | 14 | private final List outList = new ArrayList<>(); 15 | 16 | public StructObjectInspector initialize(StructObjectInspector argOIs) throws UDFArgumentException {List fieldNames = new ArrayList<>(); 17 | List fieldOIs = new ArrayList<>(); 18 | fieldNames.add("lineToWord"); 19 | fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); 20 | return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); 21 | } 22 | 23 | @Override 24 | public void process(Object[] args) throws HiveException { 25 | String arg = args[0].toString(); 26 | String splitKey = args[1].toString(); 27 | String[] fields = arg.split(splitKey); 28 | for (String field : fields) { 29 | outList.clear(); 30 | outList.add(field); 31 | forward(outList); 32 | } 33 | } 34 | 35 | @Override 36 | public void close() throws HiveException { 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /hive-learning/src/main/java/format/CustomTextInputFormat.java: -------------------------------------------------------------------------------- 1 | package format; 2 | 3 | import org.apache.hadoop.io.LongWritable; 4 | import org.apache.hadoop.io.Text; 5 | import org.apache.hadoop.mapred.*; 6 | 7 | import java.io.IOException; 8 | 9 | public class CustomTextInputFormat implements JobConfigurable, InputFormat { 10 | 11 | private final TextInputFormat format; 12 | 13 | public CustomTextInputFormat() { 14 | format = new TextInputFormat(); 15 | } 16 | 17 | @Override 18 | public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException { 19 | return format.getSplits(jobConf, numSplits); 20 | } 21 | 22 | @Override 23 | public RecordReader getRecordReader( 24 | InputSplit inputSplit, 25 | JobConf jobConf, 26 | Reporter reporter) throws IOException { 27 | reporter.setStatus(inputSplit.toString()); 28 | CustomLineRecordReader reader = new CustomLineRecordReader( 29 | new LineRecordReader(jobConf, (FileSplit) inputSplit)); 30 | reader.configure(jobConf); 31 | return reader; 32 | } 33 | 34 | @Override 35 | public void configure(JobConf jobConf) { 36 | format.configure(jobConf); 37 | } 38 | 39 | public static class CustomLineRecordReader implements 40 | RecordReader, JobConfigurable { 41 | 42 | private final LineRecordReader reader; 43 | private final Text text; 44 | 45 | public CustomLineRecordReader(LineRecordReader reader) { 46 | this.reader = reader; 47 | this.text = reader.createValue(); 48 | } 49 | 50 | @Override 51 | public void close() throws IOException { 52 | reader.close(); 53 | } 54 | 55 | @Override 56 | public LongWritable createKey() { 57 | return reader.createKey(); 58 | } 59 | 60 | @Override 61 | public Text createValue() { 62 | return new Text(); 63 | } 64 | 65 | @Override 66 | public long getPos() throws IOException { 67 | return reader.getPos(); 68 | } 69 | 70 | @Override 71 | public float getProgress() throws IOException { 72 | return reader.getProgress(); 73 | } 74 | 75 | @Override 76 | public boolean next(LongWritable key, Text value) throws IOException { 77 | while (reader.next(key, text)) { 78 | String decodeStr = text.toString().replaceAll("\\s+", ""); 79 | if (decodeStr.length() > 0) { 80 | value.set(decodeStr); 81 | return true; 82 | } 83 | } 84 | return false; 85 | } 86 | 87 | @Override 88 | public void configure(JobConf job) { } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /hive-learning/src/main/java/format/CustomTextOutputFormat.java: -------------------------------------------------------------------------------- 1 | package format; 2 | 3 | import org.apache.hadoop.fs.Path; 4 | import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; 5 | import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; 6 | import org.apache.hadoop.io.*; 7 | import org.apache.hadoop.mapred.JobConf; 8 | import org.apache.hadoop.mapred.JobConfigurable; 9 | import org.apache.hadoop.util.Progressable; 10 | 11 | import java.io.IOException; 12 | import java.util.Properties; 13 | 14 | public class CustomTextOutputFormat 15 | extends HiveIgnoreKeyTextOutputFormat { 16 | 17 | public static class CustomRecordWriter implements RecordWriter, 18 | JobConfigurable { 19 | 20 | RecordWriter writer; 21 | Text textWritable; 22 | 23 | public CustomRecordWriter(RecordWriter writer) { 24 | this.writer = writer; 25 | textWritable = new Text(); 26 | } 27 | 28 | @Override 29 | public void write(Writable w) throws IOException { 30 | textWritable.set(w.toString()); 31 | writer.write(textWritable); 32 | } 33 | 34 | @Override 35 | public void close(boolean abort) throws IOException { 36 | writer.close(abort); 37 | } 38 | 39 | @Override 40 | public void configure(JobConf job) { } 41 | } 42 | 43 | @Override 44 | public RecordWriter getHiveRecordWriter(JobConf jc, 45 | Path finalOutPath, 46 | Class valueClass, 47 | boolean isCompressed, 48 | Properties tableProperties, 49 | Progressable progress) throws IOException { 50 | CustomRecordWriter writer = new CustomRecordWriter( 51 | super.getHiveRecordWriter(jc, finalOutPath, Text.class, 52 | isCompressed, tableProperties, progress)); 53 | writer.configure(jc); 54 | return writer; 55 | } 56 | } -------------------------------------------------------------------------------- /kafka-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.apache.kafka 8 | kafka-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 15 | 2.4.1 16 | 17 | 18 | 19 | 20 | org.apache.kafka 21 | kafka-clients 22 | ${kafka.version} 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /kafka-learning/src/main/java/org/apache/kafka/learning/transaction/ConsumerExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.kafka.learning.transaction; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.clients.consumer.ConsumerRecords; 6 | import org.apache.kafka.clients.consumer.KafkaConsumer; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | 9 | import java.time.Duration; 10 | import java.util.Collections; 11 | import java.util.Properties; 12 | 13 | public class ConsumerExample { 14 | 15 | public static void main(String[] args) throws InterruptedException { 16 | Properties props = new Properties(); 17 | props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 18 | props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); 19 | props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); 20 | props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test1"); 21 | props.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); 22 | props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 23 | 24 | KafkaConsumer consumer = new KafkaConsumer<>(props); 25 | consumer.subscribe(Collections.singletonList("topic1")); 26 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 27 | for (ConsumerRecord record : records) { 28 | System.out.println(record); 29 | } 30 | 31 | 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kafka-learning/src/main/java/org/apache/kafka/learning/transaction/ProducerExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.kafka.learning.transaction; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | 8 | import java.util.Properties; 9 | 10 | public class ProducerExample { 11 | 12 | public static void main(String[] args) { 13 | Properties props = new Properties(); 14 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 15 | props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName()); 16 | props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName()); 17 | props.setProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); 18 | props.setProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "test-tx"); 19 | 20 | KafkaProducer producer = new KafkaProducer<>(props); 21 | producer.initTransactions(); 22 | try { 23 | producer.beginTransaction(); 24 | producer.send(new ProducerRecord<>("topic1", "A", "A-message")); 25 | producer.send(new ProducerRecord<>("topic1", "B", "B-message")); 26 | producer.send(new ProducerRecord<>("topic2", "A", "A-message")); 27 | producer.send(new ProducerRecord<>("topic2", "B", "B-message")); 28 | Thread.sleep(1000); 29 | double t = 1 / 0; 30 | producer.send(new ProducerRecord<>("topic1", "C", "C-messgae")); 31 | producer.send(new ProducerRecord<>("topic2", "C", "C-messgae")); 32 | producer.commitTransaction(); 33 | } catch (Exception e) { 34 | e.printStackTrace(); 35 | producer.abortTransaction(); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /spark-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | spark-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 3.2.1 13 | 14 | 15 | 16 | 17 | org.apache.spark 18 | spark-core_2.12 19 | ${spark.version} 20 | provided 21 | 22 | 23 | org.apache.spark 24 | spark-sql_2.12 25 | ${spark.version} 26 | provided 27 | 28 | 29 | 30 | 31 | 32 | net.alchim31.maven 33 | scala-maven-plugin 34 | 3.2.2 35 | 36 | 37 | 38 | testCompile 39 | 40 | 41 | 42 | 43 | 44 | org.apache.maven.plugins 45 | maven-assembly-plugin 46 | 3.1.0 47 | 48 | 49 | jar-with-dependencies 50 | 51 | 52 | 53 | 54 | make-assembly 55 | package 56 | 57 | single 58 | 59 | 60 | 61 | 62 | 63 | org.apache.maven.plugins 64 | maven-compiler-plugin 65 | 66 | 8 67 | 8 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /spark-learning/src/main/java/JavaWordCount.java: -------------------------------------------------------------------------------- 1 | import org.apache.spark.SparkConf; 2 | import org.apache.spark.SparkContext; 3 | import scala.Tuple2; 4 | 5 | import org.apache.spark.api.java.JavaPairRDD; 6 | import org.apache.spark.api.java.JavaRDD; 7 | 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.regex.Pattern; 11 | 12 | public final class JavaWordCount { 13 | private static final Pattern SPACE = Pattern.compile(" "); 14 | 15 | public static void main(String[] args) throws Exception { 16 | 17 | if (args.length < 1) { 18 | System.err.println("Usage: JavaWordCount "); 19 | System.exit(1); 20 | } 21 | 22 | SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("JavaWordCount"); 23 | SparkContext sc = new SparkContext(sparkConf); 24 | 25 | JavaRDD lines = sc.textFile(args[0], 1).toJavaRDD(); 26 | 27 | JavaRDD words = lines.flatMap(s -> Arrays.asList(SPACE.split(s)).iterator()); 28 | 29 | JavaPairRDD ones = words.mapToPair(s -> new Tuple2<>(s, 1)); 30 | 31 | JavaPairRDD counts = ones.reduceByKey(Integer::sum); 32 | 33 | List> output = counts.collect(); 34 | for (Tuple2 tuple : output) { 35 | System.out.println(tuple._1() + ": " + tuple._2()); 36 | } 37 | sc.stop(); 38 | } 39 | } -------------------------------------------------------------------------------- /spark-learning/src/main/resources/inverted_index.txt: -------------------------------------------------------------------------------- 1 | 0. it is what it is 2 | 1. what is it 3 | 2. it is a banana -------------------------------------------------------------------------------- /spark-learning/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory=ERROR, console 2 | log4j.appender.console=org.apache.log4j.ConsoleAppender 3 | log4j.appender.console.target=System.err 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 6 | # Set the default spark-shell log level to ERROR. When running the spark-shell, the 7 | # log level for this class is used to overwrite the root logger's log level, so that 8 | # the user can have different defaults for the shell and regular Spark apps. 9 | log4j.logger.org.apache.spark.repl.Main=ERROR 10 | # Settings to quiet third party logs that are too verbose 11 | log4j.logger.org.spark_project.jetty=ERROR 12 | log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR 13 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR 14 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR 15 | log4j.logger.org.apache.parquet=ERROR 16 | log4j.logger.parquet=ERROR 17 | # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support 18 | log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL 19 | log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR -------------------------------------------------------------------------------- /spark-learning/src/main/scala/InvertedIndex.scala: -------------------------------------------------------------------------------- 1 | import org.apache.spark.rdd.RDD 2 | import org.apache.spark.{SparkConf, SparkContext} 3 | 4 | import java.util 5 | 6 | object InvertedIndex { 7 | 8 | def main(args: Array[String]): Unit = { 9 | if (args.length < 1) { 10 | print("Usage: InvertedIndex ") 11 | System.exit(1) 12 | } 13 | val sparkConf = new SparkConf().setAppName("InvertedIndex") 14 | val sc : SparkContext = new SparkContext(sparkConf) 15 | val textRDD : RDD[String] = sc.textFile(args(0)) 16 | val pairRDD : RDD[(String, Int)] = textRDD.flatMap(line => { 17 | val items = line.split("\\.") 18 | val id = Integer.parseInt(items(0)) 19 | val words = items(1).trim().split("\\s+") 20 | for(i <- 0 until words.length) yield (words(i), id) 21 | }) 22 | val invertedIndexRDD = pairRDD.groupByKey().map(p => { 23 | val set = new util.HashSet[Int]() 24 | p._2.foreach(i => set.add(i)) 25 | (p._1, set) 26 | }) 27 | invertedIndexRDD.collect().foreach(println) 28 | sc.stop() 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /spark-learning/src/main/scala/WordCount.scala: -------------------------------------------------------------------------------- 1 | import org.apache.spark.rdd.RDD 2 | import org.apache.spark.{SparkConf, SparkContext} 3 | 4 | object WordCount { 5 | 6 | def main(args: Array[String]): Unit = { 7 | if (args.length < 1) { 8 | print("Usage: WordCount ") 9 | System.exit(1) 10 | } 11 | val sparkConf = new SparkConf().setAppName("WordCount") 12 | val sc : SparkContext = new SparkContext(sparkConf) 13 | val fileRDD: RDD[String] = sc.textFile(args(0)) 14 | val wordRDD: RDD[String] = fileRDD.flatMap( _.split(" ") ) 15 | val word2OneRDD: RDD[(String, Int)] = wordRDD.map((_,1)) 16 | val word2CountRDD: RDD[(String, Int)] = word2OneRDD.reduceByKey(_+_) 17 | val word2Count: Array[(String, Int)] = word2CountRDD.collect() 18 | word2Count.foreach(println) 19 | sc.stop() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /sql/antlr-learning/README.md: -------------------------------------------------------------------------------- 1 | # PlayDB 2 | 3 | PlayDB包含一个简单的用Antlr解析SQL的案例, 采用了SQLite的SQL语法文件. 它解析一段SQL并判断当前SQL应该在哪个子库中进行查询. 4 | 5 | 运行方式如下: 6 | ``` 7 | # 在antlr-learning/src/main/java/org/antlr/v4/examples/playdb/parser目录下执行 8 | antlr4 -visitor -package org.antlr.v4.examples.playdb.parser SQLite.g4 9 | ``` 10 | 11 | 最后运行`PlayDB.java`即可. -------------------------------------------------------------------------------- /sql/antlr-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | antlr-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 4.9.2 13 | 14 | 15 | 16 | 17 | org.antlr 18 | antlr4 19 | ${antlr.version} 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /sql/antlr-learning/src/main/java/org/antlr/v4/examples/playdb/PlayDB.java: -------------------------------------------------------------------------------- 1 | package org.antlr.v4.examples.playdb; 2 | 3 | import org.antlr.v4.examples.playdb.parser.SQLiteLexer; 4 | import org.antlr.v4.examples.playdb.parser.SQLiteParser; 5 | import org.antlr.v4.runtime.CharStreams; 6 | import org.antlr.v4.runtime.CommonTokenStream; 7 | import org.antlr.v4.runtime.tree.ParseTree; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | public class PlayDB { 13 | 14 | private Map region2DB = new HashMap<>(); 15 | 16 | public PlayDB() { 17 | region2DB.put("SDYT", "db1"); 18 | region2DB.put("BJHD", "db2"); 19 | region2DB.put("FJXM", "db3"); 20 | region2DB.put("SZLG", "db4"); 21 | } 22 | 23 | public String getDBName(String sql) { 24 | SQLiteLexer lexer = new SQLiteLexer(CharStreams.fromString(sql)); 25 | CommonTokenStream tokens = new CommonTokenStream(lexer); 26 | 27 | SQLiteParser parser = new SQLiteParser(tokens); 28 | ParseTree tree = parser.sql_stmt(); 29 | 30 | System.out.println(tree.toStringTree(parser)); 31 | 32 | SQLVisitor visitor = new SQLVisitor(); 33 | SelectStmt select = (SelectStmt) visitor.visit(tree); 34 | 35 | String dbName = null; 36 | if (select.tableName.equals("orders")) { 37 | if (select.whereExprs != null) { 38 | for (WhereExpr expr : select.whereExprs) { 39 | if (expr.columnName.equals("cust_id") || expr.columnName.equals("order_id")) { 40 | String region = expr.value.substring(1, 5); 41 | dbName = region2DB.get(region); 42 | break; 43 | } 44 | } 45 | } 46 | } 47 | return dbName; 48 | } 49 | 50 | public static void main(String[] args) { 51 | PlayDB playDB = new PlayDB(); 52 | 53 | String sql = "select order_id from orders where cust_id = 'SDYT987645' and price > 200"; 54 | String dbName = playDB.getDBName(sql); 55 | System.out.println("db: " + dbName); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /sql/antlr-learning/src/main/java/org/antlr/v4/examples/playdb/SelectStmt.java: -------------------------------------------------------------------------------- 1 | package org.antlr.v4.examples.playdb; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Simple class to save the information of select statement. 7 | * */ 8 | public class SelectStmt { 9 | 10 | public String tableName; 11 | public List whereExprs; 12 | 13 | public SelectStmt(String tableName, List whereExprs) { 14 | this.tableName = tableName; 15 | this.whereExprs = whereExprs; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /sql/antlr-learning/src/main/java/org/antlr/v4/examples/playdb/WhereExpr.java: -------------------------------------------------------------------------------- 1 | package org.antlr.v4.examples.playdb; 2 | 3 | /** 4 | * Simple class to save the information of where expression. 5 | * */ 6 | public class WhereExpr { 7 | 8 | public String columnName; 9 | public String op; 10 | public String value; 11 | 12 | public WhereExpr(String columnName, String op, String value) { 13 | this.columnName = columnName; 14 | this.op = op; 15 | this.value = value; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /sql/antlr-learning/src/main/java/org/antlr/v4/examples/playdb/parser/SQLite.tokens: -------------------------------------------------------------------------------- 1 | SCOL=1 2 | DOT=2 3 | OPEN_PAR=3 4 | CLOSE_PAR=4 5 | COMMA=5 6 | ASSIGN=6 7 | STAR=7 8 | PLUS=8 9 | MINUS=9 10 | TILDE=10 11 | PIPE2=11 12 | DIV=12 13 | MOD=13 14 | LT2=14 15 | GT2=15 16 | AMP=16 17 | PIPE=17 18 | LT=18 19 | LT_EQ=19 20 | GT=20 21 | GT_EQ=21 22 | EQ=22 23 | NOT_EQ1=23 24 | NOT_EQ2=24 25 | K_ABORT=25 26 | K_ACTION=26 27 | K_ADD=27 28 | K_AFTER=28 29 | K_ALL=29 30 | K_ALTER=30 31 | K_ANALYZE=31 32 | K_AND=32 33 | K_AS=33 34 | K_ASC=34 35 | K_ATTACH=35 36 | K_AUTOINCREMENT=36 37 | K_BEFORE=37 38 | K_BEGIN=38 39 | K_BETWEEN=39 40 | K_BY=40 41 | K_CASCADE=41 42 | K_CASE=42 43 | K_CAST=43 44 | K_CHECK=44 45 | K_COLLATE=45 46 | K_COLUMN=46 47 | K_COMMIT=47 48 | K_CONFLICT=48 49 | K_CONSTRAINT=49 50 | K_CREATE=50 51 | K_CROSS=51 52 | K_CURRENT_DATE=52 53 | K_CURRENT_TIME=53 54 | K_CURRENT_TIMESTAMP=54 55 | K_DATABASE=55 56 | K_DEFAULT=56 57 | K_DEFERRABLE=57 58 | K_DEFERRED=58 59 | K_DELETE=59 60 | K_DESC=60 61 | K_DETACH=61 62 | K_DISTINCT=62 63 | K_DROP=63 64 | K_EACH=64 65 | K_ELSE=65 66 | K_END=66 67 | K_ESCAPE=67 68 | K_EXCEPT=68 69 | K_EXCLUSIVE=69 70 | K_EXISTS=70 71 | K_EXPLAIN=71 72 | K_FAIL=72 73 | K_FOR=73 74 | K_FOREIGN=74 75 | K_FROM=75 76 | K_FULL=76 77 | K_GLOB=77 78 | K_GROUP=78 79 | K_HAVING=79 80 | K_IF=80 81 | K_IGNORE=81 82 | K_IMMEDIATE=82 83 | K_IN=83 84 | K_INDEX=84 85 | K_INDEXED=85 86 | K_INITIALLY=86 87 | K_INNER=87 88 | K_INSERT=88 89 | K_INSTEAD=89 90 | K_INTERSECT=90 91 | K_INTO=91 92 | K_IS=92 93 | K_ISNULL=93 94 | K_JOIN=94 95 | K_KEY=95 96 | K_LEFT=96 97 | K_LIKE=97 98 | K_LIMIT=98 99 | K_MATCH=99 100 | K_NATURAL=100 101 | K_NO=101 102 | K_NOT=102 103 | K_NOTNULL=103 104 | K_NULL=104 105 | K_OF=105 106 | K_OFFSET=106 107 | K_ON=107 108 | K_OR=108 109 | K_ORDER=109 110 | K_OUTER=110 111 | K_PLAN=111 112 | K_PRAGMA=112 113 | K_PRIMARY=113 114 | K_QUERY=114 115 | K_RAISE=115 116 | K_RECURSIVE=116 117 | K_REFERENCES=117 118 | K_REGEXP=118 119 | K_REINDEX=119 120 | K_RELEASE=120 121 | K_RENAME=121 122 | K_REPLACE=122 123 | K_RESTRICT=123 124 | K_RIGHT=124 125 | K_ROLLBACK=125 126 | K_ROW=126 127 | K_SAVEPOINT=127 128 | K_SELECT=128 129 | K_SET=129 130 | K_TABLE=130 131 | K_TEMP=131 132 | K_TEMPORARY=132 133 | K_THEN=133 134 | K_TO=134 135 | K_TRANSACTION=135 136 | K_TRIGGER=136 137 | K_UNION=137 138 | K_UNIQUE=138 139 | K_UPDATE=139 140 | K_USING=140 141 | K_VACUUM=141 142 | K_VALUES=142 143 | K_VIEW=143 144 | K_VIRTUAL=144 145 | K_WHEN=145 146 | K_WHERE=146 147 | K_WITH=147 148 | K_WITHOUT=148 149 | IDENTIFIER=149 150 | NUMERIC_LITERAL=150 151 | BIND_PARAMETER=151 152 | STRING_LITERAL=152 153 | BLOB_LITERAL=153 154 | SINGLE_LINE_COMMENT=154 155 | MULTILINE_COMMENT=155 156 | SPACES=156 157 | UNEXPECTED_CHAR=157 158 | ';'=1 159 | '.'=2 160 | '('=3 161 | ')'=4 162 | ','=5 163 | '='=6 164 | '*'=7 165 | '+'=8 166 | '-'=9 167 | '~'=10 168 | '||'=11 169 | '/'=12 170 | '%'=13 171 | '<<'=14 172 | '>>'=15 173 | '&'=16 174 | '|'=17 175 | '<'=18 176 | '<='=19 177 | '>'=20 178 | '>='=21 179 | '=='=22 180 | '!='=23 181 | '<>'=24 182 | -------------------------------------------------------------------------------- /sql/antlr-learning/src/main/java/org/antlr/v4/examples/playdb/parser/SQLiteLexer.tokens: -------------------------------------------------------------------------------- 1 | SCOL=1 2 | DOT=2 3 | OPEN_PAR=3 4 | CLOSE_PAR=4 5 | COMMA=5 6 | ASSIGN=6 7 | STAR=7 8 | PLUS=8 9 | MINUS=9 10 | TILDE=10 11 | PIPE2=11 12 | DIV=12 13 | MOD=13 14 | LT2=14 15 | GT2=15 16 | AMP=16 17 | PIPE=17 18 | LT=18 19 | LT_EQ=19 20 | GT=20 21 | GT_EQ=21 22 | EQ=22 23 | NOT_EQ1=23 24 | NOT_EQ2=24 25 | K_ABORT=25 26 | K_ACTION=26 27 | K_ADD=27 28 | K_AFTER=28 29 | K_ALL=29 30 | K_ALTER=30 31 | K_ANALYZE=31 32 | K_AND=32 33 | K_AS=33 34 | K_ASC=34 35 | K_ATTACH=35 36 | K_AUTOINCREMENT=36 37 | K_BEFORE=37 38 | K_BEGIN=38 39 | K_BETWEEN=39 40 | K_BY=40 41 | K_CASCADE=41 42 | K_CASE=42 43 | K_CAST=43 44 | K_CHECK=44 45 | K_COLLATE=45 46 | K_COLUMN=46 47 | K_COMMIT=47 48 | K_CONFLICT=48 49 | K_CONSTRAINT=49 50 | K_CREATE=50 51 | K_CROSS=51 52 | K_CURRENT_DATE=52 53 | K_CURRENT_TIME=53 54 | K_CURRENT_TIMESTAMP=54 55 | K_DATABASE=55 56 | K_DEFAULT=56 57 | K_DEFERRABLE=57 58 | K_DEFERRED=58 59 | K_DELETE=59 60 | K_DESC=60 61 | K_DETACH=61 62 | K_DISTINCT=62 63 | K_DROP=63 64 | K_EACH=64 65 | K_ELSE=65 66 | K_END=66 67 | K_ESCAPE=67 68 | K_EXCEPT=68 69 | K_EXCLUSIVE=69 70 | K_EXISTS=70 71 | K_EXPLAIN=71 72 | K_FAIL=72 73 | K_FOR=73 74 | K_FOREIGN=74 75 | K_FROM=75 76 | K_FULL=76 77 | K_GLOB=77 78 | K_GROUP=78 79 | K_HAVING=79 80 | K_IF=80 81 | K_IGNORE=81 82 | K_IMMEDIATE=82 83 | K_IN=83 84 | K_INDEX=84 85 | K_INDEXED=85 86 | K_INITIALLY=86 87 | K_INNER=87 88 | K_INSERT=88 89 | K_INSTEAD=89 90 | K_INTERSECT=90 91 | K_INTO=91 92 | K_IS=92 93 | K_ISNULL=93 94 | K_JOIN=94 95 | K_KEY=95 96 | K_LEFT=96 97 | K_LIKE=97 98 | K_LIMIT=98 99 | K_MATCH=99 100 | K_NATURAL=100 101 | K_NO=101 102 | K_NOT=102 103 | K_NOTNULL=103 104 | K_NULL=104 105 | K_OF=105 106 | K_OFFSET=106 107 | K_ON=107 108 | K_OR=108 109 | K_ORDER=109 110 | K_OUTER=110 111 | K_PLAN=111 112 | K_PRAGMA=112 113 | K_PRIMARY=113 114 | K_QUERY=114 115 | K_RAISE=115 116 | K_RECURSIVE=116 117 | K_REFERENCES=117 118 | K_REGEXP=118 119 | K_REINDEX=119 120 | K_RELEASE=120 121 | K_RENAME=121 122 | K_REPLACE=122 123 | K_RESTRICT=123 124 | K_RIGHT=124 125 | K_ROLLBACK=125 126 | K_ROW=126 127 | K_SAVEPOINT=127 128 | K_SELECT=128 129 | K_SET=129 130 | K_TABLE=130 131 | K_TEMP=131 132 | K_TEMPORARY=132 133 | K_THEN=133 134 | K_TO=134 135 | K_TRANSACTION=135 136 | K_TRIGGER=136 137 | K_UNION=137 138 | K_UNIQUE=138 139 | K_UPDATE=139 140 | K_USING=140 141 | K_VACUUM=141 142 | K_VALUES=142 143 | K_VIEW=143 144 | K_VIRTUAL=144 145 | K_WHEN=145 146 | K_WHERE=146 147 | K_WITH=147 148 | K_WITHOUT=148 149 | IDENTIFIER=149 150 | NUMERIC_LITERAL=150 151 | BIND_PARAMETER=151 152 | STRING_LITERAL=152 153 | BLOB_LITERAL=153 154 | SINGLE_LINE_COMMENT=154 155 | MULTILINE_COMMENT=155 156 | SPACES=156 157 | UNEXPECTED_CHAR=157 158 | ';'=1 159 | '.'=2 160 | '('=3 161 | ')'=4 162 | ','=5 163 | '='=6 164 | '*'=7 165 | '+'=8 166 | '-'=9 167 | '~'=10 168 | '||'=11 169 | '/'=12 170 | '%'=13 171 | '<<'=14 172 | '>>'=15 173 | '&'=16 174 | '|'=17 175 | '<'=18 176 | '<='=19 177 | '>'=20 178 | '>='=21 179 | '=='=22 180 | '!='=23 181 | '<>'=24 182 | -------------------------------------------------------------------------------- /sql/calcite-learning/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LB-Yu/data-systems-learning/1813772a6469d803409534d0d328d39f1d80b611/sql/calcite-learning/README.md -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | calcite-adapter 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | calcite-adapter-hbase 13 | 14 | 15 | 1.4.14 16 | 17 | 18 | 19 | 20 | org.apache.hbase 21 | hbase-shaded-client 22 | 1.4.14 23 | 24 | 25 | 26 | net.sf.opencsv 27 | opencsv 28 | 2.3 29 | 30 | 31 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseEnumerator.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.linq4j.Enumerator; 4 | import org.apache.calcite.rel.type.RelDataType; 5 | import org.apache.calcite.rel.type.RelDataTypeField; 6 | import org.apache.calcite.sql.type.SqlTypeName; 7 | import org.apache.hadoop.hbase.client.Result; 8 | import org.apache.hadoop.hbase.client.ResultScanner; 9 | import org.apache.hadoop.hbase.util.Bytes; 10 | 11 | import java.io.IOException; 12 | import java.util.List; 13 | 14 | public class HBaseEnumerator implements Enumerator { 15 | 16 | private final ResultScanner scanner; 17 | private final RelDataType relDataType; 18 | 19 | private Result current; 20 | 21 | public HBaseEnumerator(ResultScanner scanner, RelDataType relDataType) { 22 | this.scanner = scanner; 23 | this.relDataType = relDataType; 24 | } 25 | 26 | @Override 27 | public T current() { 28 | if (current != null) { 29 | return convertRow(current); 30 | } 31 | return null; 32 | } 33 | 34 | @Override 35 | public boolean moveNext() { 36 | try { 37 | current = scanner.next(); 38 | return (current != null); 39 | } catch (IOException e) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | 44 | @Override 45 | public void reset() { 46 | throw new UnsupportedOperationException(); 47 | } 48 | 49 | @Override 50 | public void close() { 51 | scanner.close(); 52 | } 53 | 54 | private T convertRow(Result result) { 55 | int fieldCount = relDataType.getFieldCount(); 56 | Object[] row = new Object[fieldCount]; 57 | 58 | List fieldList = relDataType.getFieldList(); 59 | for (int i = 0; i < fieldCount; ++i) { 60 | RelDataTypeField field = fieldList.get(i); 61 | 62 | String name = field.getName(); 63 | SqlTypeName type = field.getType().getSqlTypeName(); 64 | switch (type) { 65 | case INTEGER: 66 | row[i] = Bytes.toInt(getColumn(result, name)); 67 | break; 68 | case VARCHAR: 69 | default: 70 | row[i] = Bytes.toString(getColumn(result, name)); 71 | break; 72 | } 73 | 74 | } 75 | return (T) row; 76 | } 77 | 78 | private byte[] getColumn(Result result, String qualifier) { 79 | return result.getValue(Bytes.toBytes("F"), Bytes.toBytes(qualifier)); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseProject.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import org.apache.calcite.plan.RelOptCluster; 5 | import org.apache.calcite.plan.RelTraitSet; 6 | import org.apache.calcite.rel.RelNode; 7 | import org.apache.calcite.rel.core.Project; 8 | import org.apache.calcite.rel.type.RelDataType; 9 | import org.apache.calcite.rex.RexNode; 10 | import org.apache.calcite.util.Pair; 11 | 12 | import java.util.List; 13 | 14 | public class HBaseProject extends Project implements HBaseRel { 15 | 16 | public HBaseProject( 17 | RelOptCluster cluster, 18 | RelTraitSet traits, 19 | RelNode input, 20 | List projects, 21 | RelDataType rowType) { 22 | super(cluster, traits, ImmutableList.of(), input, projects, rowType); 23 | assert getConvention() == HBaseRel.CONVENTION; 24 | assert getConvention() == input.getConvention(); 25 | } 26 | 27 | @Override 28 | public Project copy( 29 | RelTraitSet traitSet, 30 | RelNode input, 31 | List projects, 32 | RelDataType rowType) { 33 | return new HBaseProject(getCluster(), traitSet, input, projects, rowType); 34 | } 35 | 36 | @Override 37 | public void implement(Implementor implementor) { 38 | implementor.visitChild(0, getInput()); 39 | for (Pair pair : getNamedProjects()) { 40 | implementor.addColumn(pair.right); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseProjectTableScanRule.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.plan.RelOptRuleCall; 4 | import org.apache.calcite.plan.RelRule; 5 | import org.apache.calcite.rel.logical.LogicalProject; 6 | import org.apache.calcite.rex.RexInputRef; 7 | import org.apache.calcite.rex.RexNode; 8 | import org.immutables.value.Value; 9 | 10 | import java.util.List; 11 | 12 | @Value.Enclosing 13 | public class HBaseProjectTableScanRule 14 | extends RelRule { 15 | 16 | public HBaseProjectTableScanRule(Config config) { 17 | super(config); 18 | } 19 | 20 | @Override 21 | public void onMatch(RelOptRuleCall call) { 22 | LogicalProject project = call.rel(0); 23 | HBaseTableScan scan = call.rel(1); 24 | int[] columns = getProjectColumns(project.getProjects()); 25 | if (columns == null) { 26 | return; 27 | } 28 | call.transformTo( 29 | new HBaseTableScan( 30 | scan.getCluster(), 31 | scan.getTable(), 32 | scan.gethBaseTable(), 33 | columns)); 34 | } 35 | 36 | private static int[] getProjectColumns(List exps) { 37 | int[] columns = new int[exps.size()]; 38 | for (int i = 0; i < exps.size(); ++i) { 39 | RexNode exp = exps.get(i); 40 | if (exp instanceof RexInputRef) { 41 | columns[i] = ((RexInputRef) exp).getIndex(); 42 | } else { 43 | return null; 44 | } 45 | } 46 | return columns; 47 | } 48 | 49 | @Value.Immutable(singleton = false) 50 | public interface Config extends RelRule.Config { 51 | Config DEFAULT = ImmutableHBaseProjectTableScanRule.Config.builder() 52 | .withOperandSupplier(b0 -> 53 | b0.operand(LogicalProject.class).oneInput(b1 -> 54 | b1.operand(HBaseTableScan.class).noInputs())) 55 | .build(); 56 | 57 | @Override 58 | default HBaseProjectTableScanRule toRule() { 59 | return new HBaseProjectTableScanRule(this); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseRel.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.plan.Convention; 4 | import org.apache.calcite.plan.RelOptTable; 5 | import org.apache.calcite.rel.RelNode; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | public interface HBaseRel extends RelNode { 11 | 12 | void implement(Implementor implementor); 13 | 14 | Convention CONVENTION = new Convention.Impl("HBASE", HBaseRel.class); 15 | 16 | class Implementor { 17 | 18 | byte[] startRow; 19 | byte[] endRow; 20 | 21 | final List columns = new ArrayList<>(); 22 | 23 | RelOptTable table; 24 | HBaseTable hBaseTable; 25 | 26 | public void setStartRow(byte[] startRow) { 27 | this.startRow = startRow; 28 | } 29 | 30 | public void setEndRow(byte[] endRow) { 31 | this.endRow = endRow; 32 | } 33 | 34 | public void addColumn(String column) { 35 | columns.add(column); 36 | } 37 | 38 | public void visitChild(int ordinal, RelNode input) { 39 | assert ordinal == 0; 40 | ((HBaseRel) input).implement(this); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseRules.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | public abstract class HBaseRules { 4 | 5 | private HBaseRules() {} 6 | 7 | public static final HBaseProjectTableScanRule PROJECT_SCAN = 8 | HBaseProjectTableScanRule.Config.DEFAULT.toRule(); 9 | } 10 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseScannableTable.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.DataContext; 4 | import org.apache.calcite.jdbc.JavaTypeFactoryImpl; 5 | import org.apache.calcite.linq4j.AbstractEnumerable; 6 | import org.apache.calcite.linq4j.Enumerable; 7 | import org.apache.calcite.linq4j.Enumerator; 8 | import org.apache.calcite.schema.ScannableTable; 9 | import org.apache.hadoop.hbase.client.ResultScanner; 10 | import org.apache.hadoop.hbase.client.Scan; 11 | import org.apache.hadoop.hbase.client.Table; 12 | 13 | import java.io.IOException; 14 | 15 | public class HBaseScannableTable extends HBaseTable implements ScannableTable { 16 | 17 | public HBaseScannableTable(Table table, String fieldsInfo) { 18 | super(table, fieldsInfo); 19 | } 20 | 21 | @Override 22 | public Enumerable scan(DataContext root) { 23 | return new AbstractEnumerable() { 24 | @Override 25 | public Enumerator enumerator() { 26 | try { 27 | Scan scan = new Scan(); 28 | ResultScanner scanner = table.getScanner(scan); 29 | return new HBaseEnumerator<>( 30 | scanner, 31 | getRowType(new JavaTypeFactoryImpl())); 32 | } catch (IOException e) { 33 | throw new RuntimeException(e); 34 | } 35 | } 36 | }; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseSchema.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import au.com.bytecode.opencsv.CSVReader; 4 | import com.google.common.collect.ImmutableMap; 5 | import org.apache.calcite.schema.Table; 6 | import org.apache.calcite.schema.impl.AbstractSchema; 7 | import org.apache.calcite.util.Source; 8 | import org.apache.calcite.util.Sources; 9 | import org.apache.hadoop.conf.Configuration; 10 | import org.apache.hadoop.hbase.HBaseConfiguration; 11 | import org.apache.hadoop.hbase.TableName; 12 | import org.apache.hadoop.hbase.client.Connection; 13 | import org.apache.hadoop.hbase.client.ConnectionFactory; 14 | 15 | import java.io.File; 16 | import java.io.IOException; 17 | import java.util.Map; 18 | 19 | public class HBaseSchema extends AbstractSchema { 20 | 21 | private final Source schemaSource; 22 | private final Connection connection; 23 | 24 | public HBaseSchema(String schemaPath) throws IOException { 25 | File schemaFile = new File(schemaPath); 26 | schemaSource = Sources.of(schemaFile); 27 | 28 | Configuration conf = HBaseConfiguration.create(); 29 | connection = ConnectionFactory.createConnection(conf); 30 | } 31 | 32 | @Override 33 | protected Map getTableMap() { 34 | final ImmutableMap.Builder builder = ImmutableMap.builder(); 35 | try { 36 | CSVReader csvReader = new CSVReader(schemaSource.reader(), '|'); 37 | String[] item; 38 | while ((item = csvReader.readNext()) != null) { 39 | if (item.length == 2) { 40 | String tableName = item[0]; 41 | String fieldsInfo = item[1]; 42 | org.apache.hadoop.hbase.client.Table table = 43 | connection.getTable(TableName.valueOf(tableName)); 44 | builder.put(tableName, new HBaseTranslatableTable(table, fieldsInfo)); 45 | } 46 | } 47 | } catch (IOException e) { 48 | throw new RuntimeException(e); 49 | } 50 | return builder.build(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseSchemaFactory.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.schema.Schema; 4 | import org.apache.calcite.schema.SchemaFactory; 5 | import org.apache.calcite.schema.SchemaPlus; 6 | 7 | import java.io.IOException; 8 | import java.util.Map; 9 | 10 | /** 11 | * Factory that creates a {@link HBaseSchema}. 12 | */ 13 | public class HBaseSchemaFactory implements SchemaFactory { 14 | 15 | @Override 16 | public Schema create(SchemaPlus parentSchema, String name, Map operand) { 17 | String schemaPath = (String) operand.get("schema"); 18 | try { 19 | return new HBaseSchema(schemaPath); 20 | } catch (IOException e) { 21 | throw new RuntimeException(e); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseTableScan.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import org.apache.calcite.adapter.enumerable.*; 5 | import org.apache.calcite.linq4j.tree.Blocks; 6 | import org.apache.calcite.linq4j.tree.Expressions; 7 | import org.apache.calcite.plan.RelOptCluster; 8 | import org.apache.calcite.plan.RelOptPlanner; 9 | import org.apache.calcite.plan.RelOptTable; 10 | import org.apache.calcite.rel.core.TableScan; 11 | import org.apache.calcite.rel.type.RelDataType; 12 | import org.apache.calcite.rel.type.RelDataTypeFactory; 13 | import org.apache.calcite.rel.type.RelDataTypeField; 14 | 15 | import java.util.List; 16 | 17 | public class HBaseTableScan extends TableScan implements EnumerableRel { 18 | 19 | private final HBaseTable hBaseTable; 20 | private final int[] columns; 21 | 22 | protected HBaseTableScan( 23 | RelOptCluster cluster, 24 | RelOptTable table, 25 | HBaseTable hBaseTable, 26 | int[] columns) { 27 | super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), ImmutableList.of(), table); 28 | this.hBaseTable = hBaseTable; 29 | this.columns = columns; 30 | } 31 | 32 | public HBaseTable gethBaseTable() { 33 | return hBaseTable; 34 | } 35 | 36 | @Override 37 | public RelDataType deriveRowType() { 38 | List fieldList = table.getRowType().getFieldList(); 39 | RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder(); 40 | for (int column : columns) { 41 | builder.add(fieldList.get(column)); 42 | } 43 | return builder.build(); 44 | } 45 | 46 | @Override 47 | public void register(RelOptPlanner planner) { 48 | planner.addRule(HBaseRules.PROJECT_SCAN); 49 | } 50 | 51 | @Override 52 | public Result implement(EnumerableRelImplementor implementor, Prefer pref) { 53 | PhysType physType = PhysTypeImpl.of( 54 | implementor.getTypeFactory(), 55 | getRowType(), 56 | pref.preferArray()); 57 | 58 | List fieldList = getRowType().getFieldList(); 59 | String[] columnNames = fieldList.stream() 60 | .map(RelDataTypeField::getName) 61 | .toArray(String[]::new); 62 | 63 | return implementor.result( 64 | physType, 65 | Blocks.toBlock( 66 | Expressions.call( 67 | table.getExpression(HBaseTranslatableTable.class), 68 | "project", 69 | implementor.getRootExpression(), 70 | Expressions.constant(columnNames)))); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/java/org/apache/calcite/adapter/hbase/HBaseToEnumerableConverter.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.adapter.enumerable.*; 4 | import org.apache.calcite.linq4j.tree.BlockBuilder; 5 | import org.apache.calcite.plan.ConventionTraitDef; 6 | import org.apache.calcite.plan.RelOptCluster; 7 | import org.apache.calcite.plan.RelTraitDef; 8 | import org.apache.calcite.plan.RelTraitSet; 9 | import org.apache.calcite.rel.RelNode; 10 | import org.apache.calcite.rel.convert.ConverterImpl; 11 | import org.apache.calcite.rel.type.RelDataType; 12 | 13 | public class HBaseToEnumerableConverter 14 | extends ConverterImpl implements EnumerableRel { 15 | 16 | protected HBaseToEnumerableConverter( 17 | RelOptCluster cluster, 18 | RelTraitSet traits, 19 | RelNode input) { 20 | super(cluster, ConventionTraitDef.INSTANCE, traits, input); 21 | } 22 | 23 | @Override 24 | public Result implement(EnumerableRelImplementor implementor, Prefer pref) { 25 | final BlockBuilder list = new BlockBuilder(); 26 | final HBaseRel.Implementor hbaseImplementor = new HBaseRel.Implementor(); 27 | hbaseImplementor.visitChild(0, getInput()); 28 | final RelDataType rowType = getRowType(); 29 | final PhysType physType = PhysTypeImpl.of( 30 | implementor.getTypeFactory(), 31 | rowType, 32 | pref.prefer(JavaRowFormat.ARRAY)); 33 | 34 | return null; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/resources/model.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "defaultSchema": "hbase", 4 | "schemas": [ 5 | { 6 | "name": "hbase", 7 | "type": "custom", 8 | "factory": "org.apache.calcite.adapter.hbase.HBaseSchemaFactory", 9 | "operand": { 10 | "schema": "/home/liebing/Code/data_systems_learning/sql-recognition-learning/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/resources/schema.csv" 11 | } 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/main/resources/schema.csv: -------------------------------------------------------------------------------- 1 | test|a:int,b:string -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/test/java/org/apache/calcite/adapter/hbase/CodeGenTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.linq4j.tree.BlockBuilder; 4 | import org.apache.calcite.linq4j.tree.BlockStatement; 5 | import org.apache.calcite.linq4j.tree.Expression; 6 | import org.apache.calcite.linq4j.tree.Expressions; 7 | import org.junit.Test; 8 | 9 | public class CodeGenTest { 10 | 11 | @Test 12 | public void testBlock() { 13 | BlockBuilder builder = new BlockBuilder(); 14 | Expression field = builder.append("field", Expressions.constant(1)); 15 | 16 | System.out.println(Expressions.toString(field)); 17 | BlockStatement blockStatement = builder.toBlock(); 18 | System.out.println(blockStatement); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/test/java/org/apache/calcite/adapter/hbase/HBaseExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.calcite.avatica.util.Casing; 4 | import org.apache.calcite.config.CalciteConnectionProperty; 5 | import org.apache.calcite.util.Sources; 6 | 7 | import java.sql.*; 8 | import java.util.Properties; 9 | 10 | public class HBaseExample { 11 | 12 | public static void main(String[] args) throws Exception { 13 | String sql = "select b from test"; 14 | 15 | Connection connection = null; 16 | Statement statement = null; 17 | try { 18 | Properties info = new Properties(); 19 | info.put("model", Sources.of(HBaseExample.class.getResource("/model.json")).file().getAbsolutePath()); 20 | info.put(CalciteConnectionProperty.UNQUOTED_CASING.toString(), Casing.UNCHANGED.toString()); 21 | connection = DriverManager.getConnection("jdbc:calcite:", info); 22 | statement = connection.createStatement(); 23 | final ResultSet resultSet = statement.executeQuery(sql); 24 | ResultSetMetaData rsmd = resultSet.getMetaData(); 25 | while (resultSet.next()) { 26 | StringBuilder sb = new StringBuilder(); 27 | for (int i = 1; i <= rsmd.getColumnCount(); ++i) { 28 | Object v = resultSet.getObject(i); 29 | sb.append(v).append(","); 30 | } 31 | sb.setLength(sb.length() - 1); 32 | System.out.println(sb); 33 | } 34 | } finally { 35 | if (statement != null) { 36 | statement.close(); 37 | } 38 | if (connection != null) { 39 | connection.close(); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/calcite-adapter-hbase/src/test/java/org/apache/calcite/adapter/hbase/HBaseTools.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.adapter.hbase; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.HBaseConfiguration; 5 | import org.apache.hadoop.hbase.HColumnDescriptor; 6 | import org.apache.hadoop.hbase.HTableDescriptor; 7 | import org.apache.hadoop.hbase.TableName; 8 | import org.apache.hadoop.hbase.client.*; 9 | import org.apache.hadoop.hbase.util.Bytes; 10 | 11 | import java.io.IOException; 12 | 13 | public class HBaseTools { 14 | 15 | public static void createTestTable() throws IOException { 16 | Configuration conf = HBaseConfiguration.create(); 17 | Connection conn = ConnectionFactory.createConnection(conf); 18 | Admin admin = conn.getAdmin(); 19 | 20 | TableName tableName = TableName.valueOf("test"); 21 | HTableDescriptor tableDesc = new HTableDescriptor(tableName); 22 | HColumnDescriptor columnDesc = new HColumnDescriptor(Bytes.toBytes("F")); 23 | tableDesc.addFamily(columnDesc); 24 | admin.createTable(tableDesc); 25 | 26 | Table table = conn.getTable(tableName); 27 | 28 | Put put1 = new Put(Bytes.toBytes("row1")); 29 | put1.addColumn(Bytes.toBytes("F"), Bytes.toBytes("a"), Bytes.toBytes(1)); 30 | put1.addColumn(Bytes.toBytes("F"), Bytes.toBytes("b"), Bytes.toBytes("hello")); 31 | table.put(put1); 32 | 33 | Put put2 = new Put(Bytes.toBytes("row2")); 34 | put2.addColumn(Bytes.toBytes("F"), Bytes.toBytes("a"), Bytes.toBytes(2)); 35 | put2.addColumn(Bytes.toBytes("F"), Bytes.toBytes("b"), Bytes.toBytes("world")); 36 | table.put(put2); 37 | 38 | table.close(); 39 | conn.close(); 40 | } 41 | 42 | public static void main(String[] args) throws IOException { 43 | createTestTable(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-adapter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | calcite-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | calcite-adapter 13 | pom 14 | 15 | calcite-adapter-hbase 16 | 17 | 18 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-avatica/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | calcite-learning 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | calcite-avatica 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | 20 | 21 | org.apache.calcite.avatica 22 | avatica-core 23 | 1.23.0 24 | 25 | 26 | 27 | org.apache.calcite.avatica 28 | avatica-server 29 | 1.23.0 30 | 31 | 32 | 33 | mysql 34 | mysql-connector-java 35 | 8.0.29 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-avatica/src/main/java/org/apache/calcite/avatica/example/simple/Client.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.avatica.example.simple; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.ResultSet; 6 | import java.sql.Statement; 7 | import java.util.Properties; 8 | 9 | public class Client { 10 | 11 | public static void main(String[] args) throws Exception { 12 | Class.forName("org.apache.calcite.avatica.remote.Driver"); 13 | Properties prop = new Properties(); 14 | prop.put("serialization", "protobuf"); 15 | try (Connection conn = 16 | DriverManager.getConnection( 17 | "jdbc:avatica:remote:url=http://localhost:5888", 18 | prop)) { 19 | 20 | final Statement stmt = conn.createStatement(); 21 | final ResultSet rs = stmt.executeQuery("SELECT * FROM student_x"); 22 | while (rs.next()) { 23 | System.out.println(rs.getInt(1)); 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-avatica/src/main/java/org/apache/calcite/avatica/example/simple/ServerExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.avatica.example.simple; 2 | 3 | import org.apache.calcite.avatica.jdbc.JdbcMeta; 4 | import org.apache.calcite.avatica.remote.Driver; 5 | import org.apache.calcite.avatica.remote.LocalService; 6 | import org.apache.calcite.avatica.remote.Service; 7 | import org.apache.calcite.avatica.server.HttpServer; 8 | 9 | public class ServerExample { 10 | 11 | public static void main(String[] args) throws Exception { 12 | String url = "jdbc:mysql://localhost:3306/test"; 13 | final JdbcMeta meta = new JdbcMeta(url, "root", "0407"); 14 | final Service service = new LocalService(meta); 15 | final HttpServer server = new HttpServer.Builder<>() 16 | .withPort(5888) 17 | .withHandler(service, Driver.Serialization.PROTOBUF) 18 | .build(); 19 | server.start(); 20 | server.join(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/codegen/config.fmpp: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This file is an FMPP (http://fmpp.sourceforge.net/) configuration file to 17 | # allow clients to extend Calcite's SQL parser to support application specific 18 | # SQL statements, literals or data types. 19 | # 20 | # Calcite's parser grammar file (Parser.jj) is written in javacc 21 | # (https://javacc.org/) with Freemarker (http://freemarker.org/) variables 22 | # to allow clients to: 23 | # 1. have custom parser implementation class and package name. 24 | # 2. insert new parser method implementations written in javacc to parse 25 | # custom: 26 | # a) SQL statements. 27 | # b) literals. 28 | # c) data types. 29 | # 3. add new keywords to support custom SQL constructs added as part of (2). 30 | # 4. add import statements needed by inserted custom parser implementations. 31 | # 32 | # Parser template file (Parser.jj) along with this file are packaged as 33 | # part of the calcite-core-.jar under "codegen" directory. 34 | 35 | data: { 36 | parser: tdd(../data/Parser.tdd) 37 | } 38 | 39 | freemarkerLinks: { 40 | includes: includes/ 41 | } 42 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/codegen/includes/compoundIdentifier.ftl: -------------------------------------------------------------------------------- 1 | <#-- 2 | // Licensed to the Apache Software Foundation (ASF) under one or more 3 | // contributor license agreements. See the NOTICE file distributed with 4 | // this work for additional information regarding copyright ownership. 5 | // The ASF licenses this file to you under the Apache License, Version 2.0 6 | // (the "License"); you may not use this file except in compliance with 7 | // the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | --> 17 | 18 | <#-- 19 | Add implementations of additional parser statements, literals or 20 | data types. 21 | 22 | Example of SqlShowTables() implementation: 23 | SqlNode SqlShowTables() 24 | { 25 | ...local variables... 26 | } 27 | { 28 | 29 | ... 30 | { 31 | return SqlShowTables(...) 32 | } 33 | } 34 | --> 35 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/CalciteUtil.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example; 2 | 3 | public class CalciteUtil { 4 | 5 | public static void print(String msg, String content) { 6 | System.out.println("========"); 7 | System.out.println(msg); 8 | System.out.println(content); 9 | System.out.println("========"); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/optimizer/VolcanoPlannerExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.optimizer; 2 | 3 | import org.apache.calcite.adapter.enumerable.EnumerableConvention; 4 | import org.apache.calcite.example.schemas.HrClusteredSchema; 5 | import org.apache.calcite.plan.ConventionTraitDef; 6 | import org.apache.calcite.plan.RelOptPlanner; 7 | import org.apache.calcite.plan.RelOptUtil; 8 | import org.apache.calcite.plan.RelTraitSet; 9 | import org.apache.calcite.rel.RelNode; 10 | import org.apache.calcite.rel.rules.CoreRules; 11 | import org.apache.calcite.schema.Schema; 12 | import org.apache.calcite.schema.SchemaPlus; 13 | import org.apache.calcite.sql.fun.SqlStdOperatorTable; 14 | import org.apache.calcite.sql.parser.SqlParser; 15 | import org.apache.calcite.tools.FrameworkConfig; 16 | import org.apache.calcite.tools.Frameworks; 17 | import org.apache.calcite.tools.RelBuilder; 18 | 19 | public class VolcanoPlannerExample { 20 | 21 | public static void main(String[] args) { 22 | SchemaPlus rootSchema = Frameworks.createRootSchema(true); 23 | Schema hrSchema = new HrClusteredSchema(); 24 | rootSchema.add("hr", hrSchema); 25 | 26 | FrameworkConfig config = Frameworks.newConfigBuilder() 27 | .parserConfig(SqlParser.Config.DEFAULT) 28 | .defaultSchema(rootSchema.getSubSchema("hr")) 29 | .build(); 30 | RelBuilder builder = RelBuilder.create(config); 31 | 32 | RelNode root = builder 33 | .scan("emps") 34 | .filter( 35 | builder.call( 36 | SqlStdOperatorTable.EQUALS, 37 | builder.field("deptno"), 38 | builder.literal(10))) 39 | .build(); 40 | System.out.println(RelOptUtil.toString(root)); 41 | 42 | RelOptPlanner planner = root.getCluster().getPlanner(); 43 | planner.addRule(CoreRules.CALC_MERGE); 44 | planner.addRelTraitDef(ConventionTraitDef.INSTANCE); 45 | RelTraitSet desiredTraits = 46 | root.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); 47 | root = planner.changeTraits(root, desiredTraits); 48 | planner.setRoot(root); 49 | RelNode optimizedRoot = planner.findBestExp(); 50 | System.out.println(RelOptUtil.toString(optimizedRoot)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/overall/SimpleDataContext.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.overall; 2 | 3 | import org.apache.calcite.DataContext; 4 | import org.apache.calcite.adapter.java.JavaTypeFactory; 5 | import org.apache.calcite.jdbc.JavaTypeFactoryImpl; 6 | import org.apache.calcite.linq4j.QueryProvider; 7 | import org.apache.calcite.schema.SchemaPlus; 8 | 9 | import java.util.concurrent.atomic.AtomicBoolean; 10 | 11 | public class SimpleDataContext implements DataContext { 12 | 13 | private final SchemaPlus schema; 14 | 15 | public SimpleDataContext(SchemaPlus schema) { 16 | this.schema = schema; 17 | } 18 | 19 | @Override 20 | public SchemaPlus getRootSchema() { 21 | return schema; 22 | } 23 | 24 | @Override 25 | public JavaTypeFactory getTypeFactory() { 26 | return new JavaTypeFactoryImpl(); 27 | } 28 | 29 | @Override 30 | public QueryProvider getQueryProvider() { 31 | throw new RuntimeException("un"); 32 | } 33 | 34 | @Override 35 | public Object get(String name) { 36 | if (Variable.CANCEL_FLAG.camelName.equals(name)) { 37 | return new AtomicBoolean(false); 38 | } 39 | return null; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/overall/SimpleSchema.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.overall; 2 | 3 | import org.apache.calcite.schema.Schema; 4 | import org.apache.calcite.schema.SchemaVersion; 5 | import org.apache.calcite.schema.Table; 6 | import org.apache.calcite.schema.impl.AbstractSchema; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | public class SimpleSchema extends AbstractSchema { 12 | 13 | private final String schemaName; 14 | private final Map tableMap; 15 | 16 | private SimpleSchema(String schemaName, Map tableMap) { 17 | this.schemaName = schemaName; 18 | this.tableMap = tableMap; 19 | } 20 | 21 | public String getSchemaName() { 22 | return schemaName; 23 | } 24 | 25 | @Override 26 | public Map getTableMap() { 27 | return tableMap; 28 | } 29 | 30 | @Override 31 | public Schema snapshot(SchemaVersion version) { 32 | return this; 33 | } 34 | 35 | public static Builder newBuilder(String schemaName) { 36 | return new Builder(schemaName); 37 | } 38 | 39 | public static final class Builder { 40 | 41 | private final String schemaName; 42 | private final Map tableMap = new HashMap<>(); 43 | 44 | private Builder(String schemaName) { 45 | if (schemaName == null || schemaName.isEmpty()) { 46 | throw new IllegalArgumentException("Schema name cannot be null or empty"); 47 | } 48 | 49 | this.schemaName = schemaName; 50 | } 51 | 52 | public Builder addTable(SimpleTable table) { 53 | if (tableMap.containsKey(table.getTableName())) { 54 | throw new IllegalArgumentException("Table already defined: " + table.getTableName()); 55 | } 56 | 57 | tableMap.put(table.getTableName(), table); 58 | 59 | return this; 60 | } 61 | 62 | public SimpleSchema build() { 63 | return new SimpleSchema(schemaName, tableMap); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/overall/SimpleTableStatistic.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.overall; 2 | 3 | import org.apache.calcite.rel.RelCollation; 4 | import org.apache.calcite.rel.RelDistribution; 5 | import org.apache.calcite.rel.RelDistributionTraitDef; 6 | import org.apache.calcite.rel.RelReferentialConstraint; 7 | import org.apache.calcite.schema.Statistic; 8 | import org.apache.calcite.util.ImmutableBitSet; 9 | 10 | import java.util.Collections; 11 | import java.util.List; 12 | 13 | public class SimpleTableStatistic implements Statistic { 14 | 15 | private final long rowCount; 16 | 17 | public SimpleTableStatistic(long rowCount) { 18 | this.rowCount = rowCount; 19 | } 20 | 21 | @Override 22 | public Double getRowCount() { 23 | return (double) rowCount; 24 | } 25 | 26 | @Override 27 | public boolean isKey(ImmutableBitSet columns) { 28 | return false; 29 | } 30 | 31 | @Override 32 | public List getKeys() { 33 | return Collections.emptyList(); 34 | } 35 | 36 | @Override 37 | public List getReferentialConstraints() { 38 | return Collections.emptyList(); 39 | } 40 | 41 | @Override 42 | public List getCollations() { 43 | return Collections.emptyList(); 44 | } 45 | 46 | @Override 47 | public RelDistribution getDistribution() { 48 | return RelDistributionTraitDef.INSTANCE.getDefault(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/parser/CalciteSQLParser.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.parser; 2 | 3 | import org.apache.calcite.sql.SqlNode; 4 | import org.apache.calcite.sql.parser.SqlParseException; 5 | import org.apache.calcite.sql.parser.SqlParser; 6 | import org.apache.calcite.sql.parser.impl.CustomSqlParserImpl; 7 | 8 | public class CalciteSQLParser { 9 | 10 | public static void main(String[] args) throws SqlParseException { 11 | String ddl = "CREATE TABLE aa (id INT) WITH ('connector' = 'file')"; 12 | String sql = "select ca, cb, cc from t where cast(ca AS INT) = 10 AND YEAR() > 2000"; 13 | 14 | String expr = "1 + 1"; 15 | 16 | SqlParser.Config config = SqlParser.config() 17 | .withParserFactory(CustomSqlParserImpl.FACTORY); 18 | 19 | SqlParser parser = SqlParser.create(expr, config); 20 | SqlNode sqlNode = parser.parseExpression(); 21 | System.out.println(sqlNode); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/parser/ddl/SqlCreateTable.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.parser.ddl; 2 | 3 | import org.apache.calcite.sql.*; 4 | import org.apache.calcite.sql.parser.SqlParserPos; 5 | import org.apache.calcite.util.ImmutableNullableList; 6 | 7 | import java.util.List; 8 | import java.util.Objects; 9 | 10 | /** 11 | * Parse tree for {@code CREATE TABLE} statement. 12 | */ 13 | public class SqlCreateTable extends SqlCreate { 14 | public final SqlIdentifier name; 15 | public final SqlNodeList columnList; 16 | public final SqlNodeList propertyList; 17 | public final SqlNode query; 18 | 19 | private static final SqlOperator OPERATOR = 20 | new SqlSpecialOperator("CREATE TABLE", SqlKind.CREATE_TABLE); 21 | 22 | /** 23 | * Creates a SqlCreateTable. 24 | */ 25 | public SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, 26 | SqlIdentifier name, SqlNodeList columnList, SqlNodeList propertyList, 27 | SqlNode query) { 28 | super(OPERATOR, pos, replace, ifNotExists); 29 | this.name = Objects.requireNonNull(name); 30 | this.columnList = columnList; // may be null 31 | this.propertyList = propertyList; // may be null 32 | this.query = query; // for "CREATE TABLE ... AS query"; may be null 33 | } 34 | 35 | public List getOperandList() { 36 | return ImmutableNullableList.of(name, columnList, propertyList, query); 37 | } 38 | 39 | @Override 40 | public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { 41 | writer.keyword("CREATE"); 42 | writer.keyword("TABLE"); 43 | if (ifNotExists) { 44 | writer.keyword("IF NOT EXISTS"); 45 | } 46 | name.unparse(writer, leftPrec, rightPrec); 47 | if (columnList != null) { 48 | SqlWriter.Frame frame = writer.startList("(", ")"); 49 | for (SqlNode c : columnList) { 50 | writer.sep(","); 51 | c.unparse(writer, 0, 0); 52 | } 53 | writer.endList(frame); 54 | } 55 | if (propertyList != null) { 56 | writer.keyword("WITH"); 57 | SqlWriter.Frame frame = writer.startList("(", ")"); 58 | for (SqlNode p : propertyList) { 59 | writer.sep(","); 60 | p.unparse(writer, 0, 0); 61 | } 62 | writer.endList(frame); 63 | } 64 | if (query != null) { 65 | writer.keyword("AS"); 66 | writer.newlineAndIndent(); 67 | query.unparse(writer, 0, 0); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/parser/ddl/SqlTableOption.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.parser.ddl; 2 | 3 | import org.apache.calcite.sql.*; 4 | import org.apache.calcite.sql.parser.SqlParserPos; 5 | import org.apache.calcite.util.ImmutableNullableList; 6 | import org.apache.calcite.util.NlsString; 7 | 8 | import java.util.List; 9 | 10 | import static java.util.Objects.requireNonNull; 11 | 12 | /** 13 | * Table options of a DDL, a key-value pair with both key and value as string literal. 14 | */ 15 | public class SqlTableOption extends SqlCall { 16 | /** 17 | * Use this operator only if you don't have a better one. 18 | */ 19 | protected static final SqlOperator OPERATOR = 20 | new SqlSpecialOperator("TableOption", SqlKind.OTHER); 21 | 22 | private final SqlNode key; 23 | private final SqlNode value; 24 | 25 | public SqlTableOption(SqlNode key, SqlNode value, SqlParserPos pos) { 26 | super(pos); 27 | this.key = requireNonNull(key, "Option key is missing"); 28 | this.value = requireNonNull(value, "Option value is missing"); 29 | } 30 | 31 | public SqlNode getKey() { 32 | return key; 33 | } 34 | 35 | public SqlNode getValue() { 36 | return value; 37 | } 38 | 39 | public String getKeyString() { 40 | return ((NlsString) SqlLiteral.value(key)).getValue(); 41 | } 42 | 43 | public String getValueString() { 44 | return ((NlsString) SqlLiteral.value(value)).getValue(); 45 | } 46 | 47 | @Override 48 | public SqlOperator getOperator() { 49 | return OPERATOR; 50 | } 51 | 52 | @Override 53 | public List getOperandList() { 54 | return ImmutableNullableList.of(key, value); 55 | } 56 | 57 | @Override 58 | public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { 59 | key.unparse(writer, leftPrec, rightPrec); 60 | writer.keyword("="); 61 | value.unparse(writer, leftPrec, rightPrec); 62 | } 63 | } 64 | 65 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/example/pretty/SQLPrettyExample.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.example.pretty; 2 | 3 | import org.apache.calcite.config.Lex; 4 | import org.apache.calcite.sql.SqlNode; 5 | import org.apache.calcite.sql.SqlWriterConfig; 6 | import org.apache.calcite.sql.dialect.FlinkSqlDialect; 7 | import org.apache.calcite.sql.parser.SqlParseException; 8 | import org.apache.calcite.sql.parser.SqlParser; 9 | import org.apache.calcite.sql.pretty.FlinkSqlPrettyWriter; 10 | import org.apache.calcite.sql.pretty.SqlPrettyWriter; 11 | import org.apache.calcite.sql.validate.SqlConformance; 12 | import org.apache.flink.sql.parser.validate.FlinkSqlConformance; 13 | import org.apache.flink.table.planner.delegation.FlinkSqlParserFactories; 14 | 15 | public class SQLPrettyExample { 16 | 17 | public static void main(String[] args) throws SqlParseException { 18 | String sql = "insert into tt select `ca`, " + 19 | "cast(cb as int), " + 20 | "conv(cc, 16, 10), " + 21 | "case when aa = 1 then 1 else 2 end " + 22 | "from t1 join t2 on t1.id = t2.id and t1.ip = t2.ip " + 23 | "where cast(ca AS INT) = 10 AND YEAR() > 2000"; 24 | 25 | SqlConformance flinkSqlConformance = FlinkSqlConformance.DEFAULT; 26 | SqlParser.Config flinkSqlConfig = SqlParser.config() 27 | .withParserFactory(FlinkSqlParserFactories.create(flinkSqlConformance)) 28 | .withConformance(flinkSqlConformance) 29 | .withLex(Lex.JAVA) 30 | .withIdentifierMaxLength(256); 31 | SqlParser parser = SqlParser.create(sql, flinkSqlConfig); 32 | SqlNode sqlNode = parser.parseStmt(); 33 | 34 | SqlWriterConfig writerConfig = SqlPrettyWriter.config() 35 | .withDialect(FlinkSqlDialect.DEFAULT) 36 | .withCaseClausesOnNewLines(false) 37 | .withIndentation(2) 38 | .withClauseStartsLine(true) 39 | .withFromFolding(SqlWriterConfig.LineFolding.TALL) 40 | .withQuoteAllIdentifiers(true) 41 | .withSelectFolding(SqlWriterConfig.LineFolding.TALL) 42 | .withSelectListItemsOnSeparateLines(true) 43 | .withWhereFolding(SqlWriterConfig.LineFolding.TALL) 44 | .withWhereListItemsOnSeparateLines(true) 45 | .withSelectListExtraIndentFlag(true) 46 | .withLineFolding(SqlWriterConfig.LineFolding.TALL) 47 | .withSelectFolding(SqlWriterConfig.LineFolding.TALL) 48 | .withClauseStartsLine(true) 49 | .withClauseEndsLine(true); 50 | SqlPrettyWriter prettyWriter = new FlinkSqlPrettyWriter(writerConfig); 51 | 52 | System.out.println(prettyWriter.format(sqlNode)); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/sql/SqlSubmit.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.sql; 2 | 3 | import org.apache.calcite.sql.parser.SqlParserPos; 4 | import org.apache.calcite.sql.util.SqlVisitor; 5 | import org.apache.calcite.sql.validate.SqlValidator; 6 | import org.apache.calcite.sql.validate.SqlValidatorScope; 7 | import org.apache.calcite.util.Litmus; 8 | 9 | public class SqlSubmit extends SqlNode { 10 | 11 | private final String jobString; 12 | 13 | public SqlSubmit(SqlParserPos pos, String jobString) { 14 | super(pos); 15 | this.jobString = jobString; 16 | } 17 | 18 | public String getJobString() { 19 | System.out.println("getJobString"); 20 | return jobString; 21 | } 22 | 23 | @Override 24 | public SqlNode clone(SqlParserPos sqlParserPos) { 25 | return null; 26 | } 27 | 28 | @Override 29 | public void unparse(SqlWriter sqlWriter, int i, int i1) { 30 | sqlWriter.keyword("submit"); 31 | sqlWriter.keyword("job"); 32 | sqlWriter.print("\n"); 33 | } 34 | 35 | @Override 36 | public void validate(SqlValidator sqlValidator, SqlValidatorScope sqlValidatorScope) { 37 | 38 | } 39 | 40 | @Override 41 | public R accept(SqlVisitor sqlVisitor) { 42 | return null; 43 | } 44 | 45 | @Override 46 | public boolean equalsDeep(SqlNode sqlNode, Litmus litmus) { 47 | return false; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/sql/dialect/FlinkSqlDialect.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.sql.dialect; 2 | 3 | import org.apache.calcite.config.NullCollation; 4 | import org.apache.calcite.sql.SqlDialect; 5 | import org.apache.flink.sql.parser.validate.FlinkSqlConformance; 6 | 7 | public class FlinkSqlDialect extends SqlDialect { 8 | 9 | public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT 10 | .withDatabaseProductName("Flink") 11 | .withConformance(FlinkSqlConformance.DEFAULT) 12 | .withIdentifierQuoteString("`") 13 | .withNullCollation(NullCollation.LOW); 14 | 15 | public static final SqlDialect DEFAULT = new SparkSqlDialect(DEFAULT_CONTEXT); 16 | 17 | public FlinkSqlDialect(Context context) { 18 | super(context); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/java/org/apache/calcite/sql/pretty/FlinkSqlPrettyWriter.java: -------------------------------------------------------------------------------- 1 | package org.apache.calcite.sql.pretty; 2 | 3 | import org.apache.calcite.sql.SqlNode; 4 | import org.apache.calcite.sql.SqlWriterConfig; 5 | import org.apache.flink.sql.parser.dml.RichSqlInsert; 6 | 7 | public class FlinkSqlPrettyWriter extends SqlPrettyWriter { 8 | 9 | public FlinkSqlPrettyWriter(SqlWriterConfig config) { 10 | super(config); 11 | } 12 | 13 | @Override 14 | public String format(SqlNode node) { 15 | assert frame == null; 16 | if (node instanceof RichSqlInsert) { 17 | RichSqlInsert richSqlInsert = (RichSqlInsert) node; 18 | 19 | Frame selectFrame = this.startList(FrameTypeEnum.SELECT); 20 | String insertKeyword = "INSERT INTO"; 21 | if (richSqlInsert.isUpsert()) { 22 | insertKeyword = "UPSERT INTO"; 23 | } else if (richSqlInsert.isOverwrite()) { 24 | insertKeyword = "INSERT OVERWRITE"; 25 | } 26 | 27 | this.sep(insertKeyword); 28 | int opLeft = richSqlInsert.getOperator().getLeftPrec(); 29 | int opRight = richSqlInsert.getOperator().getRightPrec(); 30 | richSqlInsert.getTargetTable().unparse(this, opLeft, opRight); 31 | if (richSqlInsert.getStaticPartitions() != null && 32 | richSqlInsert.getStaticPartitions().size() > 0) { 33 | this.keyword("PARTITION"); 34 | richSqlInsert.getStaticPartitions().unparse(this, opLeft, opRight); 35 | this.newlineAndIndent(); 36 | } 37 | 38 | if (richSqlInsert.getTargetColumnList() != null) { 39 | richSqlInsert.getTargetColumnList().unparse(this, opLeft, opRight); 40 | } 41 | 42 | this.newlineAndIndent(); 43 | this.endList(selectFrame); 44 | richSqlInsert.getSource().unparse(this, 0, 0); 45 | } else { 46 | node.unparse(this, 0, 0); 47 | } 48 | assert frame == null; 49 | return toString(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/resources/order.csv: -------------------------------------------------------------------------------- 1 | id:string,user_id:string,goods:string,price:double 2 | 001,1,Cola,3.5 3 | 002,1,Hat,38.9 4 | 003,2,Shoes,199.9 5 | 004,3,Book,39.9 6 | 005,4,Phone,2499.9 -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/resources/sample.txt: -------------------------------------------------------------------------------- 1 | id:int,value:string 2 | 1,A 3 | 2,B 4 | 3,C 5 | 4,D -------------------------------------------------------------------------------- /sql/calcite-learning/calcite-parser/src/main/resources/user.csv: -------------------------------------------------------------------------------- 1 | id:string,name:string,age:int 2 | 1,Jack,28 3 | 2,John,21 4 | 3,Tom,32 5 | 4,Peter,24 -------------------------------------------------------------------------------- /sql/calcite-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | calcite-learning 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | calcite-parser 13 | calcite-avatica 14 | calcite-adapter 15 | 16 | 17 | 18 | 1.32.0 19 | 30.1.1-jre 20 | 1.17.0 21 | 22 | 23 | 24 | 25 | org.apache.calcite 26 | calcite-core 27 | ${calcite.version} 28 | 29 | 30 | org.apache.calcite 31 | calcite-server 32 | ${calcite.version} 33 | 34 | 35 | org.apache.calcite 36 | calcite-file 37 | ${calcite.version} 38 | 39 | 40 | 41 | org.apache.flink 42 | flink-table-api-java 43 | ${flink.version} 44 | 45 | 46 | org.apache.flink 47 | flink-table-api-java-bridge 48 | ${flink.version} 49 | 50 | 51 | org.apache.flink 52 | flink-table-planner_2.12 53 | ${flink.version} 54 | 55 | 56 | 57 | com.google.guava 58 | guava 59 | ${guava.version} 60 | 61 | 62 | 63 | junit 64 | junit 65 | 4.13.2 66 | test 67 | 68 | 69 | 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-compiler-plugin 75 | 76 | 8 77 | 8 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /sql/javacc-learning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | javacc-learning 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 15 | 16 | 17 | 18 | 19 | org.codehaus.mojo 20 | javacc-maven-plugin 21 | 2.6 22 | 23 | 24 | 25 | javacc 26 | 27 | 28 | 29 | 30 | ${basedir}/src/main/codegen 31 | 32 | **/*.jj 33 | 34 | 2 35 | false 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/codegen/Adder.jj: -------------------------------------------------------------------------------- 1 | 2 | options { 3 | STATIC = false; 4 | } 5 | 6 | PARSER_BEGIN(Adder) 7 | import java.io.*; 8 | 9 | public class Adder { 10 | 11 | public static void main(String[] args){ 12 | for (String arg : args) { 13 | try{ 14 | System.out.println(evaluate(arg)); 15 | } catch (ParseException e) { 16 | e.printStackTrace(); 17 | } 18 | } 19 | } 20 | 21 | public static long evaluate(String src) throws ParseException { 22 | Reader reader = new StringReader(src); 23 | return new Adder(reader).expr(); 24 | } 25 | } 26 | 27 | PARSER_END(Adder) 28 | 29 | SKIP : { <[" ", "\t", "\r", "\n"]> } 30 | 31 | TOKEN : { } 32 | 33 | long expr(): 34 | { 35 | Token x, y; 36 | } 37 | { 38 | x= "+" y= 39 | { 40 | return Long.parseLong(x.image) + Long.parseLong(y.image); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/codegen/Calculator.jj: -------------------------------------------------------------------------------- 1 | 2 | options { 3 | STATIC=false; 4 | } 5 | 6 | PARSER_BEGIN(Calculator) 7 | package javacc.learning.calculator.parser; 8 | 9 | import javacc.learning.calculator.ast.*; 10 | 11 | public class Calculator { 12 | 13 | public Node parse() throws ParseException { 14 | return expr(); 15 | } 16 | } 17 | PARSER_END(Calculator) 18 | 19 | SKIP : 20 | { 21 | " " 22 | | "\r" 23 | | "\t" 24 | } 25 | 26 | TOKEN: 27 | { 28 | < NUMBER: ()+ ( "." ()+ )? > 29 | | < DIGIT: ["0"-"9"] > 30 | | < EOL: "\n" > 31 | } 32 | 33 | TOKEN: 34 | { 35 | 36 | | 37 | | 38 | } 39 | 40 | Node expr(): 41 | { 42 | Node left; 43 | Node right; 44 | } 45 | { 46 | left=term() 47 | ( 48 | "+" right=expr() { return new ExprNode(left, right, Operator.PLUS); } 49 | | "-" right=expr() { return new ExprNode(left, right, Operator.MINUS); } 50 | )* 51 | { return left; } 52 | } 53 | 54 | Node term(): 55 | { 56 | Node left; 57 | Node right; 58 | } 59 | { 60 | left=primary() 61 | ( 62 | "*" right=term() { return new TermNode(left, right, Operator.MUL); } 63 | | "/" right=term() { return new TermNode(left, right, Operator.DIV); } 64 | )* 65 | { return left; } 66 | } 67 | 68 | Node primary(): 69 | { 70 | Token t; 71 | Token p; 72 | Node n; 73 | } 74 | { 75 | LOOKAHEAD( "!") 76 | t= "!" 77 | { 78 | String value = t.image; 79 | double number = Double.parseDouble(value); 80 | return new FactorialNode(new ValueNode(number)); 81 | } 82 | | 83 | t= 84 | { 85 | double number = Double.parseDouble(t.image); 86 | return new ValueNode(number); 87 | } 88 | | LOOKAHEAD("(" n=expr() ")" "!") 89 | "(" n=expr() ")" "!" 90 | { 91 | return new FactorialNode(n); 92 | } 93 | | "+" n=primary() { return n; } 94 | | "-" n=primary() 95 | { 96 | n.setSign(-1); 97 | return n; 98 | } 99 | | "(" n=expr() ")" 100 | { 101 | return n; 102 | } 103 | | "(" n=expr() ")" 104 | { 105 | return new SinNode(n); 106 | } 107 | | "(" n=expr() ")" 108 | { 109 | return new CosNode(n); 110 | } 111 | | "(" n=expr() ")" 112 | { 113 | return new TanNode(n); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/Main.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator; 2 | 3 | import javacc.learning.calculator.ast.Node; 4 | import javacc.learning.calculator.parser.Calculator; 5 | import javacc.learning.calculator.parser.ParseException; 6 | import javacc.learning.calculator.visitor.CalculateVisitor; 7 | import javacc.learning.calculator.visitor.DumpVisitor; 8 | 9 | public class Main { 10 | 11 | public static void main(String[] args) throws ParseException { 12 | Calculator calculator = new Calculator(System.in); 13 | Node node = calculator.parse(); 14 | 15 | CalculateVisitor calculateVisitor = new CalculateVisitor(); 16 | System.out.println(calculateVisitor.calculate(node)); 17 | 18 | System.out.println("\nAST dump:"); 19 | DumpVisitor dumpVisitor = new DumpVisitor(); 20 | dumpVisitor.dump(node); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/CosNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class CosNode extends UnaryNode { 6 | 7 | public CosNode(Node node) { 8 | this.node = node; 9 | } 10 | 11 | @Override 12 | public T accept(ASTVisitor visitor) { 13 | return visitor.visit(this); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/ExprNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class ExprNode extends Node { 6 | 7 | private final Node left; 8 | private final Node right; 9 | private final Operator op; 10 | 11 | public ExprNode(Node left, Node right, Operator op) { 12 | this.left = left; 13 | this.right = right; 14 | this.op = op; 15 | } 16 | 17 | public Node getLeft() { 18 | return left; 19 | } 20 | 21 | public Node getRight() { 22 | return right; 23 | } 24 | 25 | public Operator getOp() { 26 | return op; 27 | } 28 | 29 | @Override 30 | public T accept(ASTVisitor visitor) { 31 | return visitor.visit(this); 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return (sign == 1 ? "" : "-") + String.format("%s %s %s", left, op, right); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/FactorialNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class FactorialNode extends UnaryNode { 6 | 7 | public FactorialNode(Node node) { 8 | this.node = node; 9 | } 10 | 11 | @Override 12 | public String toString() { 13 | if (node instanceof ValueNode) { 14 | return (sign == 1 ? "" : "-") + node + "!"; 15 | } else { 16 | return (sign == 1 ? "" : "-") + "(" + node + ")" + "!"; 17 | } 18 | } 19 | 20 | @Override 21 | public T accept(ASTVisitor visitor) { 22 | return visitor.visit(this); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/Node.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public abstract class Node { 6 | 7 | protected int sign = 1; 8 | 9 | public int getSign() { 10 | return sign; 11 | } 12 | 13 | public void setSign(int sign) { 14 | this.sign = sign; 15 | } 16 | 17 | public abstract T accept(ASTVisitor visitor); 18 | } 19 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/Operator.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | public enum Operator { 4 | 5 | PLUS("+"), 6 | MINUS("-"), 7 | MUL("*"), 8 | DIV("/"); 9 | 10 | private final String symbol; 11 | 12 | Operator(String symbol) { 13 | this.symbol = symbol; 14 | } 15 | 16 | @Override 17 | public String toString() { 18 | return symbol; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/SinNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class SinNode extends UnaryNode { 6 | 7 | public SinNode(Node node) { 8 | this.node = node; 9 | } 10 | 11 | @Override 12 | public T accept(ASTVisitor visitor) { 13 | return visitor.visit(this); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/TanNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class TanNode extends UnaryNode { 6 | 7 | public TanNode(Node node) { 8 | this.node = node; 9 | } 10 | 11 | @Override 12 | public T accept(ASTVisitor visitor) { 13 | return visitor.visit(this); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/TermNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class TermNode extends Node { 6 | 7 | private final Node left; 8 | private final Node right; 9 | private final Operator op; 10 | 11 | public TermNode(Node left, Node right, Operator op) { 12 | this.left = left; 13 | this.right = right; 14 | this.op = op; 15 | } 16 | 17 | public Node getLeft() { 18 | return left; 19 | } 20 | 21 | public Node getRight() { 22 | return right; 23 | } 24 | 25 | public Operator getOp() { 26 | return op; 27 | } 28 | 29 | @Override 30 | public T accept(ASTVisitor visitor) { 31 | return visitor.visit(this); 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return (sign == 1 ? "" : "-") + String.format("%s %s %s", left.toString(), op, right.toString()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/UnaryNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | public abstract class UnaryNode extends Node { 4 | 5 | protected Node node; 6 | 7 | public Node getNode() { 8 | return node; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/ast/ValueNode.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.ast; 2 | 3 | import javacc.learning.calculator.visitor.ASTVisitor; 4 | 5 | public class ValueNode extends Node { 6 | 7 | protected double value; 8 | 9 | public ValueNode(double value) { 10 | this.value = value; 11 | } 12 | 13 | public double getValue() { 14 | return value; 15 | } 16 | 17 | @Override 18 | public T accept(ASTVisitor visitor) { 19 | return visitor.visit(this); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return (sign == 1 ? "" : "-") + value; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/visitor/ASTVisitor.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.visitor; 2 | 3 | import javacc.learning.calculator.ast.*; 4 | 5 | public interface ASTVisitor { 6 | 7 | T visit(ExprNode node); 8 | 9 | T visit(TermNode node); 10 | 11 | T visit(SinNode node); 12 | 13 | T visit(CosNode node); 14 | 15 | T visit(TanNode node); 16 | 17 | T visit(FactorialNode node); 18 | 19 | T visit(ValueNode node); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/visitor/CalculateVisitor.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.visitor; 2 | 3 | import javacc.learning.calculator.ast.*; 4 | 5 | public class CalculateVisitor implements ASTVisitor { 6 | 7 | public double calculate(Node node) { 8 | return node.accept(this); 9 | } 10 | 11 | @Override 12 | public Double visit(ExprNode node) { 13 | double leftValue = node.getLeft().accept(this); 14 | double rightValue = node.getRight().accept(this); 15 | switch (node.getOp()) { 16 | case PLUS: 17 | return (leftValue + rightValue) * node.getSign(); 18 | case MINUS: 19 | return (leftValue - rightValue) * node.getSign(); 20 | default: 21 | throw new IllegalArgumentException("Illegal operator for expr node"); 22 | } 23 | } 24 | 25 | @Override 26 | public Double visit(TermNode node) { 27 | double leftValue = node.getLeft().accept(this); 28 | double rightValue = node.getRight().accept(this); 29 | switch (node.getOp()) { 30 | case MUL: 31 | return leftValue * rightValue * node.getSign(); 32 | case DIV: 33 | return leftValue / rightValue * node.getSign(); 34 | default: 35 | throw new IllegalArgumentException("Illegal operator for term node"); 36 | } 37 | } 38 | 39 | @Override 40 | public Double visit(SinNode node) { 41 | return Math.sin(node.getNode().accept(this)) * node.getSign(); 42 | } 43 | 44 | @Override 45 | public Double visit(CosNode node) { 46 | return Math.cos(node.getNode().accept(this)) * node.getSign(); 47 | } 48 | 49 | @Override 50 | public Double visit(TanNode node) { 51 | return Math.tan(node.getNode().accept(this)) * node.getSign(); 52 | } 53 | 54 | @Override 55 | public Double visit(FactorialNode node) { 56 | double value = node.getNode().accept(this); 57 | double result = 1; 58 | for (int i = 1; i <= value; ++i) { 59 | result *= i; 60 | } 61 | return result * node.getSign(); 62 | } 63 | 64 | @Override 65 | public Double visit(ValueNode node) { 66 | return node.getValue() * node.getSign(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /sql/javacc-learning/src/main/java/javacc/learning/calculator/visitor/DumpVisitor.java: -------------------------------------------------------------------------------- 1 | package javacc.learning.calculator.visitor; 2 | 3 | import javacc.learning.calculator.ast.*; 4 | 5 | public class DumpVisitor implements ASTVisitor { 6 | 7 | private int level = 0; 8 | 9 | public void dump(Node node) { 10 | node.accept(this); 11 | } 12 | 13 | @Override 14 | public Void visit(ExprNode node) { 15 | printIndent(level); 16 | System.out.println(node.getOp()); 17 | 18 | ++level; 19 | node.getLeft().accept(this); 20 | node.getRight().accept(this); 21 | --level; 22 | return null; 23 | } 24 | 25 | @Override 26 | public Void visit(TermNode node) { 27 | printIndent(level); 28 | System.out.println(node.getOp()); 29 | 30 | ++level; 31 | node.getLeft().accept(this); 32 | node.getRight().accept(this); 33 | --level; 34 | return null; 35 | } 36 | 37 | @Override 38 | public Void visit(SinNode node) { 39 | printIndent(level); 40 | System.out.println("sin"); 41 | 42 | ++level; 43 | node.getNode().accept(this); 44 | --level; 45 | return null; 46 | } 47 | 48 | @Override 49 | public Void visit(CosNode node) { 50 | printIndent(level); 51 | System.out.println("cos"); 52 | 53 | ++level; 54 | node.getNode().accept(this); 55 | --level; 56 | return null; 57 | } 58 | 59 | @Override 60 | public Void visit(TanNode node) { 61 | printIndent(level); 62 | System.out.println("tan"); 63 | 64 | ++level; 65 | node.getNode().accept(this); 66 | --level; 67 | return null; 68 | } 69 | 70 | @Override 71 | public Void visit(FactorialNode node) { 72 | printIndent(level); 73 | System.out.println("!"); 74 | 75 | ++level; 76 | node.getNode().accept(this); 77 | --level; 78 | return null; 79 | } 80 | 81 | @Override 82 | public Void visit(ValueNode node) { 83 | printIndent(level); 84 | System.out.println(node); 85 | return null; 86 | } 87 | 88 | public void printIndent(int level) { 89 | for (int i = 0; i < level * 2; ++i) { 90 | System.out.print(" "); 91 | } 92 | } 93 | } 94 | --------------------------------------------------------------------------------