├── README.md ├── disct ├── chain.conf ├── sql.jar ├── student.txt └── teacher.txt ├── doc └── img │ ├── comm.png │ ├── sql1.png │ ├── sql2.png │ ├── sql3.png │ ├── sql4.png │ ├── sql5.png │ └── sql6.png └── src └── sql-egine ├── .classpath ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── pom.xml ├── src ├── main │ ├── java │ │ ├── com │ │ │ ├── conf │ │ │ │ ├── SqlChain.java │ │ │ │ └── SqlConf.java │ │ │ ├── engine │ │ │ │ ├── MrEngine.java │ │ │ │ └── SqlEngine.java │ │ │ ├── file │ │ │ │ ├── FileTable.java │ │ │ │ ├── HDFSTable.java │ │ │ │ └── Table.java │ │ │ ├── hdfs │ │ │ │ └── HDFSHelper.java │ │ │ ├── mr │ │ │ │ ├── SortMapper.java │ │ │ │ ├── SortReducer.java │ │ │ │ ├── SqlCombiner.java │ │ │ │ ├── SqlMapper.java │ │ │ │ └── SqlReducer.java │ │ │ └── sql │ │ │ │ ├── SqlExeEngine.java │ │ │ │ └── SqlParse.java │ │ └── org │ │ │ └── apache │ │ │ └── hadoop │ │ │ └── io │ │ │ └── nativeio │ │ │ ├── NativeIO.java │ │ │ └── NativeIO.java.bak │ └── resources │ │ ├── chain.conf │ │ ├── log4j.properties │ │ ├── sql.conf │ │ ├── student.txt │ │ └── teacher.txt └── test │ ├── java │ └── com │ │ └── file │ │ ├── FileTableTest.java │ │ └── FileTableTest.java.bak │ └── resources │ ├── log4j.properties │ ├── student.txt │ ├── student.txt.bak │ ├── teacher.txt │ └── teacher.txt.bak └── target ├── classes ├── chain.conf ├── com │ ├── conf │ │ ├── SqlChain.class │ │ └── SqlConf.class │ ├── engine │ │ ├── MrEngine.class │ │ └── SqlEngine.class │ ├── file │ │ ├── FileTable.class │ │ ├── HDFSTable.class │ │ └── Table.class │ ├── hdfs │ │ └── HDFSHelper.class │ ├── mr │ │ ├── SortMapper.class │ │ ├── SortReducer.class │ │ ├── SqlCombiner.class │ │ ├── SqlMapper.class │ │ └── SqlReducer.class │ └── sql │ │ ├── SqlExeEngine$1.class │ │ ├── SqlExeEngine$1On.class │ │ ├── SqlExeEngine.class │ │ └── SqlParse.class ├── log4j.properties ├── org │ └── apache │ │ └── hadoop │ │ └── io │ │ └── nativeio │ │ ├── NativeIO$CachedUid.class │ │ ├── NativeIO$POSIX$CacheManipulator.class │ │ ├── NativeIO$POSIX$CachedName.class │ │ ├── NativeIO$POSIX$IdCache.class │ │ ├── NativeIO$POSIX$NoMlockCacheManipulator.class │ │ ├── NativeIO$POSIX$Stat.class │ │ ├── NativeIO$POSIX.class │ │ ├── NativeIO$Windows$AccessRight.class │ │ ├── NativeIO$Windows.class │ │ ├── NativeIO.class │ │ └── NativeIO.java.bak ├── sql.conf ├── student.txt └── teacher.txt └── test-classes ├── com └── file │ ├── FileTableTest.class │ └── FileTableTest.java.bak ├── log4j.properties ├── student.txt ├── student.txt.bak ├── teacher.txt └── teacher.txt.bak /README.md: -------------------------------------------------------------------------------- 1 | # SQL Engine 2 | 3 | 4 | 5 | --- 6 | 7 | # 一、简介 8 | 闲来无事简单是写了写，如何在HDFS上执行SQL，不像Hive不需要MySQL支持，只需要简单配置一下需要执行的SQL和日志的格式，同时支持对日志进行正则过滤和提取必要字段。支持JSON格式的日志。 9 | 10 | # 二、配置需要执行的SQL和日志格式 11 | ```shell 12 | ############## 13 | # HDFS 14 | ############## 15 | log.hdfs=hdfs://192.168.1.32:9000 16 | 17 | ############## 18 | # SQL 19 | ############## 20 | 21 | # 简单查询 22 | log.sql1=create /sql/out1 as select * from s order by id desc 23 | 24 | # 简单查询 25 | log.sql2=create /sql/out2 as select id,name,grade from s where id>10 order by grade desc limit 0,10 26 | 27 | # 查询最高的成绩 28 | log.sql3=create /sql/out3 as select max(grade) as grade from s 29 | 30 | # 表连接 31 | log.sql4=create /sql/out4 as select s.id,s.name,s.grade,t.id,t.name from s join t on s.tid=t.id limit 0,10 32 | 33 | # 分组查询 34 | log.sql5=create /sql/out5 as select s.tid,count(s.id) as s.count from s group by s.tid 35 | 36 | # 表连接分组查询 37 | log.sql6=create /sql/out6 as select t.name,count(t.id) as t.count from s join t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,5 38 | 39 | # log chain 40 | log.chain=sql1,sql2,sql3,sql4,sql5,sql6 41 | 42 | ############## 43 | # VAR 44 | ############## 45 | # log table 46 | log.table.t=/sql/teacher.txt:id|name:#split:#filter 47 | log.table.s=/sql/student.txt:id|name|grade|tid:#split:#filter 48 | 49 | # split 50 | log.split=| 51 | 52 | # log filter 53 | log.filter=(^[^#].*) 54 | ``` 55 | 56 | # 三、执行 57 | 58 | ```shell 59 | # 执行命令 60 | hadoop jar sql.jar com.engine.MrEngine /log/chain.conf 61 | ``` 62 | ![comm][1] 63 | ```shell 64 | # 简单查询 65 | select * from s order by id desc 66 | ``` 67 | ![sql1][2] 68 | ```shell 69 | # 简单查询 70 | select id,name,grade from s where id>10 order by grade desc limit 0,10 71 | ``` 72 | ![sql2][3] 73 | ```shell 74 | # 查询最高的成绩 75 | select max(grade) as grade from s 76 | ``` 77 | ![sql3][4] 78 | ```shell 79 | # 表连接 80 | select s.id,s.name,s.grade,t.id,t.name from s join t on s.tid=t.id limit 0,10 81 | ``` 82 | ![sql4][5] 83 | ```shell 84 | # 分组查询 85 | select s.tid,count(s.id) as s.count from s group by s.tid 86 | ``` 87 | ![sql5][6] 88 | ```shell 89 | # 表连接分组查询 90 | select t.name,count(t.id) as t.count from s join t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,5 91 | ``` 92 | ![sql6][7] 93 | 94 | [1]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/comm.png 95 | [2]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql1.png 96 | [3]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql2.png 97 | [4]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql3.png 98 | [5]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql4.png 99 | [6]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql5.png 100 | [7]: https://raw.githubusercontent.com/mircode/sql-engine/master/doc/img/sql6.png 101 | -------------------------------------------------------------------------------- /disct/chain.conf: -------------------------------------------------------------------------------- 1 | ############## 2 | # HDFS 3 | ############## 4 | log.hdfs=hdfs://192.168.1.32:9000 5 | 6 | ############## 7 | # SQL 8 | ############## 9 | 10 | # 简单查询 11 | log.sql1=create /sql/out1 as select * from s order by id desc 12 | 13 | # 简单查询 14 | log.sql2=create /sql/out2 as select id,name,grade from s where id>10 order by grade desc limit 0,10 15 | 16 | # 查询最高的成绩 17 | log.sql3=create /sql/out3 as select max(grade) as grade from s 18 | 19 | # 表连接 20 | log.sql4=create /sql/out4 as select s.id,s.name,s.grade,t.id,t.name from s join t on s.tid=t.id limit 0,10 21 | 22 | # 分组查询 23 | log.sql5=create /sql/out5 as select s.tid,count(s.id) as s.count from s group by s.tid 24 | 25 | # 表连接分组查询 26 | log.sql6=create /sql/out6 as select t.name,count(t.id) as t.count from s join t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,5 27 | 28 | # log chain 29 | log.chain=sql1,sql2,sql3,sql4,sql5,sql6 30 | 31 | ############## 32 | # VAR 33 | ############## 34 | # log table 35 | log.table.t=/sql/teacher.txt:id|name:#split:#filter 36 | log.table.s=/sql/student.txt:id|name|grade|tid:#split:#filter 37 | 38 | # split 39 | log.split=| 40 | 41 | # log filter 42 | log.filter=(^[^#].*) -------------------------------------------------------------------------------- /disct/sql.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/disct/sql.jar -------------------------------------------------------------------------------- /disct/student.txt: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /disct/teacher.txt: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /doc/img/comm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/comm.png -------------------------------------------------------------------------------- /doc/img/sql1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql1.png -------------------------------------------------------------------------------- /doc/img/sql2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql2.png -------------------------------------------------------------------------------- /doc/img/sql3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql3.png -------------------------------------------------------------------------------- /doc/img/sql4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql4.png -------------------------------------------------------------------------------- /doc/img/sql5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql5.png -------------------------------------------------------------------------------- /doc/img/sql6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/doc/img/sql6.png -------------------------------------------------------------------------------- /src/sql-egine/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/sql-egine/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | sql-egine 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/sql-egine/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding/=UTF-8 3 | -------------------------------------------------------------------------------- /src/sql-egine/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate 4 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 5 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 6 | org.eclipse.jdt.core.compiler.compliance=1.8 7 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 8 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 9 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 10 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 12 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 13 | org.eclipse.jdt.core.compiler.source=1.8 14 | -------------------------------------------------------------------------------- /src/sql-egine/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /src/sql-egine/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | sql-egine 4 | sql-egine 5 | 0.0.1-SNAPSHOT 6 | 7 | 2.7.1 8 | 2.3.1 9 | 10 | 11 | 12 | jdk.tools 13 | jdk.tools 14 | 1.8 15 | system 16 | ${JAVA_HOME}/lib/tools.jar 17 | 18 | 19 | org.apache.hadoop 20 | hadoop-common 21 | ${hadoop.version} 22 | 23 | 24 | org.apache.hadoop 25 | hadoop-hdfs 26 | ${hadoop.version} 27 | 28 | 29 | org.apache.hadoop 30 | hadoop-mapreduce-client-core 31 | ${hadoop.version} 32 | 33 | 34 | org.apache.hadoop 35 | hadoop-mapreduce-client-jobclient 36 | ${hadoop.version} 37 | 38 | 39 | org.apache.hadoop 40 | hadoop-mapreduce-client-common 41 | ${hadoop.version} 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/conf/SqlChain.java: -------------------------------------------------------------------------------- 1 | package com.conf; 2 | 3 | import java.io.FileInputStream; 4 | import java.net.URLDecoder; 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Properties; 11 | 12 | import com.file.Table; 13 | import com.sql.SqlParse; 14 | 15 | public class SqlChain { 16 | 17 | // 配置前缀 18 | public static final String LOG_PREFIX = "log"; 19 | // 变量前缀 20 | public static final String LOG_VAR_PREFIX = "#"; 21 | // HDFS路径 22 | public static final String LOG_HDFS = "log.hdfs"; 23 | // SQL执行链 24 | public static final String LOG_CHAIN = "log.chain"; 25 | 26 | public static List getChain(String path) { 27 | 28 | if(path.startsWith("classpath:")){ 29 | path=getResource(path); 30 | } 31 | Properties prop = new Properties(); 32 | 33 | // 读取配置文件 34 | try { 35 | FileInputStream in = new FileInputStream(path); 36 | prop.load(in); 37 | in.close(); 38 | } catch (Exception e) { 39 | e.printStackTrace(); 40 | } 41 | // 解析基本配置 42 | String hdfs = prop.getProperty(SqlChain.LOG_HDFS); 43 | String chain = prop.getProperty(SqlChain.LOG_CHAIN, 44 | SqlChain.LOG_VAR_PREFIX + "sql"); 45 | 46 | // 　保存配置信息 47 | List confs = new ArrayList(); 48 | 49 | // 解析变量 50 | for (String sql : chain.split(",")) { 51 | 52 | if(!sql.startsWith("#")) sql="#"+sql; 53 | sql=SqlChain.get(prop, sql); 54 | 55 | // 解析SQL 56 | SqlParse sqlParse = new SqlParse(sql); 57 | 58 | // 解析SQL的表结构 59 | Map tables = new HashMap(); 60 | 61 | // 解析SQL的输出路径 62 | String output = sqlParse.get(SqlParse.OUTPUT); 63 | output = SqlChain.get(prop, output); 64 | 65 | // 解析SQL的输入路径 66 | String inputs = sqlParse.get(SqlParse.INPUT); 67 | // input:name或name 68 | for (String input : inputs.split(",")) { 69 | 70 | String name = null; 71 | // 解析出表名 72 | if (input.contains(":")) { 73 | name = input.split(":")[1]; 74 | input = input.replace(":", "."); 75 | } else { 76 | name = input; 77 | input = "*." + input; 78 | } 79 | // 解析输入 80 | String value = SqlChain.get(prop, input); 81 | String[] splits = value.split(":"); 82 | 83 | String in = null; 84 | if (splits.length >= 1) { 85 | in = SqlChain.get(prop, splits[0]); 86 | } 87 | String format = null; 88 | if (splits.length >= 2) { 89 | format = SqlChain.get(prop, splits[1]); 90 | } 91 | String split = null; 92 | if (splits.length >= 3) { 93 | split = SqlChain.get(prop, splits[2]); 94 | } 95 | String filter = null; 96 | if (splits.length >= 4) { 97 | filter = SqlChain.get(prop, splits[3]); 98 | } 99 | tables.put(name, new Table(name, in, split, format, filter)); 100 | } 101 | 102 | // 创建Conf对象 103 | SqlConf conf = new SqlConf(hdfs, sql, output, tables); 104 | confs.add(conf); 105 | 106 | } 107 | 108 | return confs; 109 | 110 | } 111 | 112 | /** 113 | * 获取变量的值,如果key不是变量,则直接返回key 114 | * 115 | * @param prop 116 | * @param key 117 | * #input.t或*.t 118 | * @return 119 | */ 120 | private static String get(Properties prop, String key) { 121 | if (key.startsWith(LOG_VAR_PREFIX)) { 122 | return prop.getProperty( 123 | LOG_PREFIX + "." + key.substring(1), key); 124 | } else if (key.startsWith("*")) { 125 | 126 | String value = null; 127 | String regex = key.replace(".", "\\.").replace("*", ".*"); 128 | regex = LOG_PREFIX + "\\." + regex; 129 | Iterator> it = prop.entrySet().iterator(); 130 | while (it.hasNext()) { 131 | Map.Entry entry = it.next(); 132 | String k = entry.getKey().toString(); 133 | String v = entry.getValue().toString(); 134 | if (k.matches(regex)) { 135 | value = v; 136 | break; 137 | } 138 | } 139 | return value; 140 | } else { 141 | return key; 142 | } 143 | } 144 | 145 | public static String getResource(String resource) { 146 | String path = SqlConf.class.getClassLoader().getResource(resource.substring("classpath:".length())) 147 | .getPath(); 148 | try { 149 | path = URLDecoder.decode(path, "UTF-8"); 150 | } catch (Exception e) { 151 | e.printStackTrace(); 152 | } 153 | return path; 154 | } 155 | 156 | 157 | 158 | } 159 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/conf/SqlConf.java: -------------------------------------------------------------------------------- 1 | package com.conf; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | 8 | import com.file.Table; 9 | import com.sql.SqlExeEngine; 10 | import com.sql.SqlParse; 11 | 12 | /** 13 | * 解析配置文件 14 | * 15 | * @author 魏国兴 16 | * 17 | */ 18 | public class SqlConf extends Configuration { 19 | 20 | // 配置文件的属性 21 | public static final String CONF_SQL = "#sql"; 22 | 23 | // 输入,输出,临时目录 24 | public static final String CONF_TMP = "#tmp"; 25 | public static final String CONF_INPUT = "#input"; 26 | public static final String CONF_OUTPUT = "#output"; 27 | 28 | // 是否需要排序 29 | public static final String CONF_SORT = "#isSort"; 30 | // 是否需要分组 31 | public static final String CONF_GROUP = "#isGroup"; 32 | 33 | // Join临时表 34 | public static final String CONF_JOINTABLE = "#joinTable"; 35 | 36 | // HDFS路径 37 | public static final String CONF_HDFS = "fs.defaultFS"; 38 | 39 | // 解析SQL 40 | public SqlParse sqlParse = null; 41 | 42 | // SQL语句 43 | private String sql = null; 44 | // hdfs地址 45 | private String hdfs = null; 46 | 47 | // 输入目录 48 | private String input = null; 49 | // 输出目录 50 | private String output = null; 51 | // 临时目录 52 | private String tmp = null; 53 | 54 | // 是否需要执行排序任务 55 | private Boolean isSort = false; 56 | // 是否需要执行分组 57 | private Boolean isGroup = false; 58 | // Join之后表结构 59 | private String joinTable = null; 60 | // Table映射 61 | private Map tables = new HashMap(); 62 | 63 | public SqlConf() { 64 | } 65 | 66 | public SqlConf(String hdfs, String sql, String output, 67 | Map tables) { 68 | 69 | // 解析输入输入和输出目录 70 | this.sqlParse = new SqlParse(sql); 71 | 72 | this.hdfs = hdfs; 73 | this.sql = sql; 74 | this.output = output; 75 | this.tables = tables; 76 | 77 | // 初始化临时目录 78 | initInput(); 79 | 80 | // 计算Join之后的表结构 81 | initJoin(); 82 | 83 | // 判断是否需要排序或分组 84 | initSort(); 85 | initGroup(); 86 | 87 | // 如果不需要排序和分组,那么临时目录就是输出目录 88 | if (!isSort || !isGroup) { 89 | this.tmp = this.output; 90 | } 91 | 92 | // 将变量加载到Context中 93 | loadContext(); 94 | 95 | } 96 | 97 | /** 98 | * 加载SQL和表映射到Context中 99 | */ 100 | public void loadContext() { 101 | 102 | // 加载SQL 103 | this.set(CONF_SQL, this.sql); 104 | 105 | // 执行SQL中的临时,输入,输出目录 106 | this.set(CONF_TMP, this.tmp); 107 | this.set(CONF_INPUT, this.input); 108 | this.set(CONF_OUTPUT, this.output); 109 | 110 | // 是否需要排序和分组 111 | this.setBoolean(CONF_GROUP, this.isGroup); 112 | this.setBoolean(CONF_SORT, this.isSort); 113 | 114 | // Join时的表结构 115 | if(joinTable!=null){ 116 | this.set(CONF_JOINTABLE, this.joinTable); 117 | } 118 | 119 | // 加载HFDS地址 120 | this.set(CONF_HDFS, this.hdfs); 121 | 122 | // 加载其他表结构 123 | for (Map.Entry entry : this.tables.entrySet()) { 124 | String ser = entry.getValue().serialize(); 125 | this.set(entry.getKey(), ser); 126 | } 127 | } 128 | 129 | // 是否需要排序 130 | private void initSort() { 131 | this.isSort = false; 132 | String distinct = sqlParse.get(SqlParse.DISTINCT); 133 | String order = sqlParse.get(SqlParse.ORDER); 134 | String limit = sqlParse.get(SqlParse.LIMIT); 135 | if (distinct != null || order != null || limit != null) 136 | this.isSort = true; 137 | } 138 | 139 | // 是否需要分组 140 | private void initGroup() { 141 | this.isGroup = false; 142 | String matrix = sqlParse.get(SqlParse.MATRIX); 143 | String group = sqlParse.get(SqlParse.GROUP); 144 | if (matrix != null || group != null) 145 | this.isGroup = true; 146 | } 147 | 148 | private void initJoin() { 149 | SqlExeEngine sqlEngine = new SqlExeEngine(tables.get(sqlParse 150 | .get(SqlParse.MAIN_TABLE))); 151 | String join = sqlParse.get(SqlParse.JOIN); 152 | // 如果有join的表,则Reduce端的表格式需要变更 153 | if (join != null) { 154 | for (String en : join.split("\\|")) { 155 | String table = en.split("on")[0]; 156 | String on = en.split("on")[1]; 157 | String name = SqlParse.getTable(table); 158 | sqlEngine.join(tables.get(name), on); 159 | } 160 | this.joinTable = sqlEngine.getTable().serialize(); 161 | } 162 | } 163 | 164 | /** 165 | * 初始化输入目录 166 | */ 167 | private void initInput() { 168 | this.input = this.tables.get(sqlParse.get(SqlParse.MAIN_TABLE)).getInput(); 169 | if (output != null) { 170 | if (output.endsWith("/")) { 171 | output = output.substring(0, this.output.length() - 1); 172 | } 173 | this.tmp = output.substring(0, output.lastIndexOf("/") + 1) 174 | + "tmp/"; 175 | } else { 176 | if (input.endsWith("/")) { 177 | this.output = input + "out/"; 178 | this.tmp = input + "tmp/"; 179 | } else { 180 | this.output = input.substring(0, input.lastIndexOf("/") + 1) 181 | + "out/"; 182 | this.tmp = input.substring(0, input.lastIndexOf("/") + 1) 183 | + "tmp/"; 184 | } 185 | } 186 | } 187 | 188 | } 189 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/engine/MrEngine.java: -------------------------------------------------------------------------------- 1 | package com.engine; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.NullWritable; 9 | import org.apache.hadoop.io.Text; 10 | import org.apache.hadoop.mapreduce.Job; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; 13 | import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl; 14 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 15 | 16 | import com.conf.SqlChain; 17 | import com.conf.SqlConf; 18 | import com.hdfs.HDFSHelper; 19 | import com.mr.SortMapper; 20 | import com.mr.SortReducer; 21 | import com.mr.SqlCombiner; 22 | import com.mr.SqlMapper; 23 | import com.mr.SqlReducer; 24 | 25 | /** 26 | * 通过Hadoop执行SQL任务 27 | * @author 魏国兴 28 | * 29 | */ 30 | public class MrEngine { 31 | 32 | public static void main(String[] args) throws Exception { 33 | // 配置文件路径 34 | String path = "classpath:chain.conf"; 35 | if(args.length>1){ 36 | path=args[1]; 37 | } 38 | // SQL引擎 39 | MrEngine engine = new MrEngine(); 40 | // 执行SQL 41 | engine.execute(path); 42 | } 43 | 44 | // 执行SQL 45 | public void execute(String path) throws IOException, 46 | ClassNotFoundException, InterruptedException { 47 | 48 | // 执行多条SQL 49 | List confs = SqlChain.getChain(path); 50 | 51 | // 链式执行SQL 52 | for (SqlConf conf : confs) { 53 | execute(conf); 54 | } 55 | // 打印执行结果 56 | for (SqlConf conf : confs) { 57 | System.out.println(conf.get(SqlConf.CONF_SQL)); 58 | // 打印执行结果 59 | System.out.println(conf.sqlParse.get("#mr.sort.format").replace(",", "|")); 60 | List result = HDFSHelper.readLines(conf, conf.get("#output") 61 | + "/part-r-00000",20); 62 | for(String row:result) 63 | System.out.println(row); 64 | } 65 | } 66 | 67 | public void execute(SqlConf conf) throws IOException, 68 | ClassNotFoundException, InterruptedException { 69 | 70 | // ######################################### 71 | // Job 控制器 72 | // ######################################### 73 | 74 | // Job控制器 75 | JobControl jobControl = new JobControl(MrEngine.class.getName() 76 | + "_JobChain"); 77 | 78 | 79 | if (conf.getBoolean(SqlConf.CONF_GROUP, false)) { 80 | Job mainJob = this.groupJob(conf); 81 | // 主Job 82 | ControlledJob cmainJob = new ControlledJob( 83 | mainJob.getConfiguration()); 84 | cmainJob.setJob(mainJob); 85 | jobControl.addJob(cmainJob); 86 | 87 | Job sortJob = this.sortJob(conf); 88 | if (conf.getBoolean(SqlConf.CONF_SORT, false)) { 89 | // 排序Job 90 | ControlledJob csortJob = new ControlledJob( 91 | sortJob.getConfiguration()); 92 | csortJob.setJob(sortJob); 93 | csortJob.addDependingJob(cmainJob); 94 | jobControl.addJob(csortJob); 95 | } 96 | } else { 97 | Job filterJob = this.filterJob(conf); 98 | // 主Job 99 | ControlledJob cfilterJob = new ControlledJob( 100 | filterJob.getConfiguration()); 101 | cfilterJob.setJob(filterJob); 102 | jobControl.addJob(cfilterJob); 103 | } 104 | // 运行Job 105 | new Thread(jobControl).start(); 106 | 107 | // 等待执行完成,并打印执行进度 108 | while (!jobControl.allFinished()) { 109 | Thread.sleep(500); 110 | for (ControlledJob job : jobControl.getRunningJobList()) { 111 | this.display(job.getJob()); 112 | } 113 | } 114 | jobControl.stop(); 115 | } 116 | 117 | public void display(Job job) { 118 | try { 119 | // JobConf conf=(JobConf)job.getConfiguration(); 120 | System.out.printf("Job " + job.getJobName() 121 | + ": map: %.1f%% reduce %.1f%%\n", 122 | 100.0 * job.mapProgress(), 100.0 * job.reduceProgress()); 123 | // System.out.println("Job total maps = " + conf.getNumMapTasks()); 124 | // System.out.println("Job total reduces = " + 125 | // conf.getNumReduceTasks()); 126 | // System.out.flush(); 127 | } catch (Exception e) { 128 | } 129 | 130 | } 131 | 132 | public Job filterJob(Configuration conf) throws IOException { 133 | // ######################################### 134 | // 分组Job 135 | // ######################################### 136 | 137 | // 清除输出目录 138 | HDFSHelper.deleteOnExit(conf, conf.get(SqlConf.CONF_OUTPUT)); 139 | 140 | // 设置输入 141 | Path in = new Path(conf.get(SqlConf.CONF_INPUT)); 142 | Path out = new Path(conf.get(SqlConf.CONF_OUTPUT)); 143 | 144 | Job job = Job.getInstance(conf, MrEngine.class.getName() + "_filter"); 145 | job.setJarByClass(MrEngine.class); 146 | 147 | FileInputFormat.setInputPaths(job, in); 148 | FileOutputFormat.setOutputPath(job, out); 149 | 150 | job.setMapperClass(SqlMapper.class); 151 | job.setReducerClass(SortReducer.class); 152 | 153 | job.setMapOutputKeyClass(Text.class); 154 | job.setMapOutputValueClass(Text.class); 155 | 156 | job.setOutputKeyClass(NullWritable.class); 157 | job.setOutputValueClass(Text.class); 158 | 159 | // System.exit(mainJob.waitForCompletion(true)?0:1); 160 | 161 | return job; 162 | } 163 | 164 | public Job groupJob(Configuration conf) throws IOException { 165 | // ######################################### 166 | // 分组Job 167 | // ######################################### 168 | 169 | // 清除输出目录 170 | HDFSHelper.deleteOnExit(conf, conf.get(SqlConf.CONF_TMP)); 171 | 172 | // 设置输入 173 | Path in = new Path(conf.get(SqlConf.CONF_INPUT)); 174 | Path out = new Path(conf.get(SqlConf.CONF_TMP)); 175 | 176 | Job job = Job.getInstance(conf, MrEngine.class.getName() + "_main"); 177 | job.setJarByClass(MrEngine.class); 178 | FileInputFormat.setInputPaths(job, in); 179 | FileOutputFormat.setOutputPath(job, out); 180 | 181 | job.setMapperClass(SqlMapper.class); 182 | job.setReducerClass(SqlReducer.class); 183 | job.setCombinerClass(SqlCombiner.class); 184 | 185 | job.setMapOutputKeyClass(Text.class); 186 | job.setMapOutputValueClass(Text.class); 187 | 188 | job.setOutputKeyClass(NullWritable.class); 189 | job.setOutputValueClass(Text.class); 190 | 191 | // System.exit(mainJob.waitForCompletion(true)?0:1); 192 | 193 | return job; 194 | } 195 | 196 | public Job sortJob(Configuration conf) throws IOException { 197 | 198 | // ######################################### 199 | // 排序 Job 200 | // ######################################### 201 | 202 | // 清除输出目录 203 | HDFSHelper.deleteOnExit(conf, conf.get(SqlConf.CONF_OUTPUT)); 204 | 205 | // 设置输入 206 | Path sortIn = new Path(conf.get(SqlConf.CONF_TMP)); 207 | Path sortOut = new Path(conf.get(SqlConf.CONF_OUTPUT)); 208 | 209 | Job job = Job.getInstance(conf, MrEngine.class.getName() + "_sort"); 210 | job.setJarByClass(MrEngine.class); 211 | FileInputFormat.setInputPaths(job, sortIn); 212 | FileOutputFormat.setOutputPath(job, sortOut); 213 | 214 | job.setMapperClass(SortMapper.class); 215 | job.setReducerClass(SortReducer.class); 216 | 217 | job.setMapOutputKeyClass(Text.class); 218 | job.setMapOutputValueClass(Text.class); 219 | 220 | job.setOutputKeyClass(NullWritable.class); 221 | job.setOutputValueClass(Text.class); 222 | 223 | return job; 224 | } 225 | 226 | } 227 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/engine/SqlEngine.java: -------------------------------------------------------------------------------- 1 | package com.engine; 2 | 3 | import com.file.FileTable; 4 | import com.file.Table; 5 | import com.sql.SqlExeEngine; 6 | import com.sql.SqlParse; 7 | 8 | public class SqlEngine { 9 | 10 | // 操作表 11 | private Table table = null; 12 | // 操作SQL 13 | private SqlParse sqlParse = null; 14 | 15 | public SqlExeEngine SQL(String sql) { 16 | 17 | SqlExeEngine engine = new SqlExeEngine(table); 18 | 19 | this.sqlParse = new SqlParse(sql); 20 | 21 | // 获取到主表 22 | String from = sqlParse.get(SqlParse.FROM); 23 | if (from != null) { 24 | 25 | String name = SqlParse.getTable(from); 26 | String input = SqlParse.getPath(from); 27 | 28 | // 从input中解析出format和split 29 | this.table = new FileTable(name, input, null, null,null); 30 | engine.setTable(table); 31 | } 32 | 33 | // 执行join 34 | String join = sqlParse.get(SqlParse.JOIN); 35 | if (join != null) { 36 | 37 | for (String en : join.split("\\|")) { 38 | String table = en.split("on")[0]; 39 | String on = en.split("on")[1]; 40 | 41 | String name = SqlParse.getTable(table); 42 | String input = SqlParse.getPath(table); 43 | 44 | engine.join(new FileTable(name, input, null, null,null), on); 45 | } 46 | } 47 | 48 | // 执行where 49 | String where = sqlParse.get(SqlParse.WHERE); 50 | if (where != null) { 51 | engine.where(where); 52 | } 53 | 54 | // 执行group by 55 | String matrix = sqlParse.get(SqlParse.MATRIX); 56 | String group = sqlParse.get(SqlParse.GROUP); 57 | if (group != null || matrix != null) { 58 | engine.group(matrix, group); 59 | } 60 | 61 | // 执行过滤 62 | String select = sqlParse.get(SqlParse.SELECT); 63 | if (select != null) { 64 | engine.select(select); 65 | } 66 | String distinct = sqlParse.get(SqlParse.DISTINCT); 67 | if (distinct != null) { 68 | engine.distinct(distinct); 69 | } 70 | // 执行order by 71 | String order = sqlParse.get(SqlParse.ORDER); 72 | if (order != null) { 73 | engine.order(order); 74 | } 75 | // 执行limit 76 | String limit = sqlParse.get(SqlParse.LIMIT); 77 | if (limit != null) { 78 | engine.limit(limit); 79 | } 80 | 81 | return engine; 82 | } 83 | 84 | public static void main(String args[]) { 85 | 86 | // 声明变量 87 | String sql = null; 88 | SqlExeEngine sqlEngine = null; 89 | 90 | Table student = new FileTable("s","classpath:student.txt"); 91 | Table teacher = new FileTable("t","classpath:teacher.txt");; 92 | 93 | // 简单查询 94 | sql = "select id,name,grade from classpath:student.txt student where id>10 order by grade desc limit 0,10"; 95 | 96 | sqlEngine = new SqlEngine().SQL(sql); 97 | System.out.println(sqlEngine); 98 | 99 | sqlEngine = new SqlExeEngine(student).where("id>10").order("grade desc") 100 | .limit("0,10").select("id,name,grade"); 101 | 102 | System.out.println(sqlEngine); 103 | 104 | // 查询最高的成绩 105 | sql = "select max(grade) as grade from classpath:student.txt s"; 106 | 107 | sqlEngine = new SqlEngine().SQL(sql); 108 | System.out.println(sqlEngine); 109 | 110 | student = new FileTable("classpath:student.txt"); 111 | String max = new SqlExeEngine(student).max("grade"); 112 | System.out.println("grade"); 113 | System.out.println(max); 114 | 115 | // 表连接 116 | sql = "select s.id,s.name,s.grade,t.id,t.name from classpath:student.txt s join classpath:teacher.txt t on s.tid=t.id limit 0,10"; 117 | 118 | sqlEngine = new SqlEngine().SQL(sql); 119 | System.out.println(sqlEngine); 120 | 121 | teacher = new FileTable("t", "classpath:teacher.txt"); 122 | student = new FileTable("s", "classpath:student.txt"); 123 | 124 | sqlEngine = new SqlExeEngine(student).join(teacher, "s.tid=t.id") 125 | .select("s.id,s.name,s.grade,t.id,t.name").limit("0,10"); 126 | System.out.println(sqlEngine); 127 | 128 | // 分组查询 129 | sql = "select tid,sum(grade) as grade from classpath:student.txt student group by tid limit 0,10"; 130 | 131 | sqlEngine = new SqlEngine().SQL(sql); 132 | System.out.println(sqlEngine); 133 | 134 | student = new FileTable("s", "classpath:student.txt"); 135 | sqlEngine = new SqlExeEngine(student).group("sum(grade) as grade", "tid") 136 | .limit("0,10"); 137 | System.out.println(sqlEngine); 138 | 139 | // 表连接分组查询(查询那个老师对应的学生比较多) 140 | sql = "select t.name,count(t.id) as t.count from classpath:student.txt s join classpath:teacher.txt t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,10"; 141 | 142 | sqlEngine = new SqlEngine().SQL(sql); 143 | System.out.println(sqlEngine); 144 | 145 | teacher = new FileTable("t", "classpath:teacher.txt"); 146 | student = new FileTable("s", "classpath:student.txt"); 147 | 148 | sqlEngine = new SqlExeEngine(student).join(teacher, "s.tid=t.id") 149 | .group("count(t.id) as t.count", "t.id,t.name") 150 | .select("t.name,count(t.id) as t.count").order("t.count desc") 151 | .limit("0,10"); 152 | System.out.println(sqlEngine); 153 | 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/file/FileTable.java: -------------------------------------------------------------------------------- 1 | package com.file; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.InputStreamReader; 7 | import java.net.URLDecoder; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | import com.conf.SqlConf; 12 | 13 | /** 14 | * 从文件中加载Table 15 | * 16 | * @author 魏国兴 17 | * 18 | */ 19 | public class FileTable extends Table { 20 | 21 | 22 | public FileTable(String input) { 23 | this(null, input, null, null, null); 24 | } 25 | 26 | public FileTable(String name, String input) { 27 | this(name, input, null, null, null); 28 | } 29 | 30 | public FileTable(String name, String input, String split, String format) { 31 | this(name, input, split, format, null); 32 | } 33 | 34 | public FileTable(String name, String split, String format, List rows) { 35 | this(name, null, split, format, rows); 36 | } 37 | 38 | public FileTable(String name, String input, String split, String format, 39 | List rows) { 40 | 41 | if (name == null) { 42 | name = new File(input).getName(); 43 | if (name.lastIndexOf(".") > -1) { 44 | name = name.substring(0, name.lastIndexOf(".")); 45 | } 46 | } 47 | if(input.startsWith("classpath:")){ 48 | input=getResource(input); 49 | } 50 | if (split == null) 51 | split = this.readSplit(input); 52 | if (format == null) 53 | format = this.readFormat(input); 54 | 55 | this.name = name; 56 | this.input = input; 57 | this.split = split; 58 | this.format = format; 59 | if (rows != null) { 60 | this.addRows(rows); 61 | } 62 | this.addRows(this.readRows(input)); 63 | 64 | } 65 | 66 | /** 67 | * 从文件中获取日志格式 68 | * 69 | * @param path 70 | * @return 71 | */ 72 | public String readFormat(String path) { 73 | String format = null; 74 | 75 | // 读取一行 76 | List rows = readFile(path, null, 1); 77 | 78 | if (rows != null) { 79 | format = rows.get(0); 80 | if (format.startsWith("#")) { 81 | format = format.substring("#".length()); 82 | } else { 83 | String spl = this.readSplit(path); 84 | String splits[] = format.split(spl); 85 | String fmt = ""; 86 | for (int i = 1; i <= splits.length; i++) { 87 | fmt += "col" + i + spl; 88 | } 89 | if (!fmt.equals("")) { 90 | format = fmt.substring(0, fmt.length() - 1); 91 | } 92 | } 93 | } 94 | return format; 95 | } 96 | 97 | /** 98 | * 从文件中获取分隔串 99 | * 100 | * @param path 101 | * @return 102 | */ 103 | public String readSplit(String path) { 104 | 105 | String split = "|"; 106 | 107 | // 从文件中读取3行 108 | List rows = readFile(path, null, 2); 109 | 110 | String header = rows.get(0); 111 | String line = rows.get(1); 112 | 113 | // 尝试使用常用分隔符切分记录 114 | String[] splits = { "\\|", ",", "\\s+", "-", "_", ":" }; 115 | 116 | for (String regex : splits) { 117 | int len1 = header.split(regex).length; 118 | int len2 = line.split(regex).length; 119 | 120 | if (len1 != 0 && len1 == len2) { 121 | if (regex.equals("\\s+")) { 122 | split = " "; 123 | } 124 | if (regex.equals("\\|")) { 125 | split = "|"; 126 | } else { 127 | split = regex; 128 | } 129 | break; 130 | } 131 | } 132 | // 返回分隔符 133 | return split; 134 | } 135 | 136 | /** 137 | * 从文件中获取记录数 138 | * 139 | * @param path 140 | * @return 141 | */ 142 | public List readRows(String path) { 143 | // 读取所有非#号开头的行 144 | return this.readFile(path, "[^#].*", -1); 145 | } 146 | 147 | /** 148 | * 读取文件指定行数 149 | * 150 | */ 151 | private List readFile(String path, String filter, int num) { 152 | 153 | // 读取文件内容到List中 154 | List rows = new ArrayList(); 155 | // 读取0行 156 | if (num == 0) 157 | return rows; 158 | try { 159 | // 获取文件的输入流 160 | BufferedReader reader = new BufferedReader(new InputStreamReader( 161 | new FileInputStream(URLDecoder.decode(path, "UTF-8")), 162 | "UTF-8")); 163 | // 记录行数 164 | int counter = 0; 165 | // 读取文件 166 | String line = null; 167 | while ((line = reader.readLine()) != null) { 168 | // 读取指定行数 169 | if (num > 0 && counter++ > num) { 170 | break; 171 | } else { 172 | if (filter == null 173 | || (filter != null && line.matches(filter))) { 174 | rows.add(line); 175 | } 176 | } 177 | } 178 | reader.close(); 179 | } catch (Exception e) { 180 | e.printStackTrace(); 181 | } 182 | return rows.size() == 0 ? null : rows; 183 | } 184 | private String getResource(String resource) { 185 | String path = SqlConf.class.getClassLoader().getResource(resource.substring("classpath:".length())) 186 | .getPath(); 187 | try { 188 | path = URLDecoder.decode(path, "UTF-8"); 189 | } catch (Exception e) { 190 | e.printStackTrace(); 191 | } 192 | return path; 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/file/HDFSTable.java: -------------------------------------------------------------------------------- 1 | package com.file; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | 7 | import com.hdfs.HDFSHelper; 8 | 9 | 10 | /** 11 | * 文件到表格的映射对象 12 | * 13 | * @author 魏国兴 14 | * 15 | */ 16 | public class HDFSTable extends Table{ 17 | 18 | public HDFSTable(Configuration conf,Table table){ 19 | this.name=table.getName(); 20 | this.split=table.getSplit(); 21 | this.format=table.getFormat(); 22 | this.input=table.getInput(); 23 | try { 24 | super.setRows(HDFSHelper.readLines(conf, this.input)); 25 | } catch (IOException e) { 26 | e.printStackTrace(); 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/file/Table.java: -------------------------------------------------------------------------------- 1 | package com.file; 2 | 3 | import java.io.StringWriter; 4 | import java.io.Writer; 5 | import java.util.ArrayList; 6 | import java.util.Arrays; 7 | import java.util.List; 8 | import java.util.regex.Matcher; 9 | import java.util.regex.Pattern; 10 | 11 | import org.codehaus.jackson.map.ObjectMapper; 12 | import org.codehaus.jackson.map.type.TypeFactory; 13 | 14 | /** 15 | * 文件到表格的映射对象 16 | * 17 | * @author 魏国兴 18 | * 19 | */ 20 | public class Table { 21 | 22 | // 表格名称 23 | protected String name; 24 | // 文件路径 25 | protected String input; 26 | // 文件分隔符 27 | protected String split; 28 | // 日志格式 29 | protected String format; 30 | // 行元素应该符合的正则表达式 31 | protected String filter; 32 | // 表格数据 33 | protected List rows=new ArrayList(); 34 | 35 | // 当前集合是否为空 36 | @SuppressWarnings("unused") 37 | private boolean empty; 38 | 39 | public Table(){ 40 | 41 | } 42 | 43 | public Table(String name,String input,String split,String format,String filter){ 44 | this(name,input,split,format,filter,null); 45 | } 46 | public Table(String name,String input,String split,String format,String filter,List rows){ 47 | this.name=name; 48 | this.input=input; 49 | this.split=split; 50 | this.format=format; 51 | this.filter=filter; 52 | if(rows!=null) 53 | this.setRows(rows); 54 | } 55 | 56 | 57 | 58 | /** 59 | * 根据index返回line中对应的值 60 | * 61 | * @param line 62 | * @param index 63 | * @return 64 | */ 65 | public String getColumn(String line, int index) { 66 | String[] splits = line.split(getRgxSplit()); 67 | return splits[index]; 68 | } 69 | 70 | 71 | 72 | /** 73 | * 返回col中的多个列 74 | * 75 | * @param line 76 | * @param cols 77 | * @return 78 | */ 79 | public String getColumns(String line,String cols){ 80 | 81 | String row = ""; 82 | if(cols.equals("*")){ 83 | row=line; 84 | }else{ 85 | String splits[]=cols.split(","); 86 | for (String col : splits) { 87 | row += this.getColumn(line, col,String.class)+split; 88 | } 89 | row = row.substring(0, row.length() - 1); 90 | } 91 | return row; 92 | } 93 | public String toString() { 94 | String res = this.format + "\n"; 95 | if(rows!=null){ 96 | for (String row : rows) { 97 | res += row + "\n"; 98 | } 99 | } 100 | return res; 101 | } 102 | /** 103 | * 根据col返回line中对应的值 104 | * 105 | * @param line 106 | * @param col 107 | * @return 108 | */ 109 | @SuppressWarnings("unchecked") 110 | public T getColumn(String line,String col,Class type){ 111 | String[] splits = format.split(getRgxSplit()); 112 | int index = Arrays.asList(splits).indexOf(col); 113 | if(index<0){ 114 | if(col.startsWith(name)){ 115 | index = Arrays.asList(splits).indexOf(col.substring((name+".").length())); 116 | }else{ 117 | index = Arrays.asList(splits).indexOf(name+"."+col); 118 | } 119 | } 120 | String val=this.getColumn(line, index); 121 | 122 | String name=type.getSimpleName(); 123 | 124 | Object res=null; 125 | if(name.toLowerCase().contains("double")){ 126 | res=Double.parseDouble(val); 127 | }else if(name.toLowerCase().contains("float")){ 128 | res=Float.parseFloat(val); 129 | }else if(name.toLowerCase().contains("integer")){ 130 | res=Integer.parseInt(val); 131 | }else if(name.toLowerCase().contains("boolean")){ 132 | res=Integer.parseInt(val); 133 | }else{ 134 | res=val; 135 | } 136 | return (T)res; 137 | 138 | } 139 | /** 140 | * 返回分隔符的正则表达式形式 141 | * 142 | * @return 143 | */ 144 | private String getRgxSplit() { 145 | String regex = split; 146 | if (regex.equals("|")) { 147 | regex = "\\|"; 148 | } 149 | if (regex.equals(" ")) { 150 | regex = "\\s+"; 151 | } 152 | return regex; 153 | } 154 | /** 155 | * 将FileTable序列化为Json串 156 | * @return 157 | */ 158 | public String serialize() { 159 | List rows = this.rows; 160 | this.rows = null; 161 | ObjectMapper mapper = new ObjectMapper(); 162 | Writer write = new StringWriter(); 163 | try { 164 | mapper.writeValue(write, this); 165 | } catch (Exception e) { 166 | e.printStackTrace(); 167 | } 168 | this.rows = rows; 169 | return write.toString(); 170 | 171 | } 172 | /** 173 | * Json串反序列化为FileTable 174 | * @param json 175 | * @return 176 | */ 177 | public Table diserialize(String json) { 178 | Table fileTable = null; 179 | ObjectMapper mapper = new ObjectMapper(); 180 | try { 181 | fileTable = (Table) mapper.readValue(json, 182 | TypeFactory.rawClass(Table.class)); 183 | } catch (Exception e) { 184 | e.printStackTrace(); 185 | } 186 | 187 | fileTable.setRows(new ArrayList()); 188 | return fileTable; 189 | } 190 | public String getName() { 191 | return name; 192 | } 193 | public Table setName(String name) { 194 | this.name = name; 195 | return this; 196 | } 197 | public String getInput() { 198 | return input; 199 | } 200 | public Table setInput(String input) { 201 | this.input = input; 202 | return this; 203 | } 204 | public String getSplit() { 205 | return split; 206 | } 207 | public Table setSplit(String split) { 208 | this.split = split; 209 | return this; 210 | } 211 | public String getFormat() { 212 | return format; 213 | } 214 | public Table setFormat(String format) { 215 | this.format = format; 216 | return this; 217 | } 218 | public String getFilter() { 219 | return filter; 220 | } 221 | public void setFilter(String filter) { 222 | this.filter = filter; 223 | } 224 | 225 | public List getRows() { 226 | return rows; 227 | } 228 | 229 | 230 | public Table setRows(List rows) { 231 | this.rows = rows; 232 | return this; 233 | } 234 | public Table setRow(String row) { 235 | List rows=new ArrayList(); 236 | rows.add(row); 237 | this.rows=rows; 238 | return this; 239 | } 240 | /** 241 | * 添加一行数据 242 | * @param row 243 | */ 244 | public Table addRow(String row){ 245 | this.rows.add(row); 246 | return this; 247 | } 248 | /** 249 | * 添加多行数据 250 | * @param row 251 | */ 252 | public Table addRows(List rows){ 253 | this.rows.addAll(rows); 254 | return this; 255 | } 256 | 257 | public boolean isEmpty(){ 258 | if(this.rows==null||this.rows.isEmpty()){ 259 | return true; 260 | }else{ 261 | return false; 262 | } 263 | } 264 | public Table filterRows(){ 265 | List rows=new ArrayList(); 266 | if(filter!=null&&this.rows!=null){ 267 | for(String r:this.rows){ 268 | String nw=this.filterRow(r, filter); 269 | if(nw!=null) rows.add(nw); 270 | } 271 | this.rows=rows; 272 | } 273 | return this; 274 | } 275 | public String filterRow(String r,String filter){ 276 | if(filter!=null){ 277 | // 拆分SQL 278 | Pattern p = Pattern.compile(filter); 279 | Matcher m = p.matcher(r); 280 | String row=""; 281 | 282 | // 将正则匹配的结果存放到Map中 283 | while (m.find()) { 284 | int count=m.groupCount(); 285 | if(!m.group().equals("")){ 286 | for(int i=1;i<=count;i++){ 287 | String n=m.group(i); 288 | if(n.endsWith(split)){ 289 | row+=n; 290 | }else{ 291 | row+=n+split; 292 | } 293 | } 294 | } 295 | } 296 | if(!row.equals("")){ 297 | if(row.endsWith(split)){ 298 | row=row.substring(0,row.length()-1); 299 | } 300 | }else{ 301 | row=null; 302 | } 303 | return row; 304 | } 305 | return r; 306 | } 307 | 308 | public void setEmpty(boolean empty) { 309 | this.empty = empty; 310 | } 311 | 312 | 313 | } 314 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/hdfs/HDFSHelper.java: -------------------------------------------------------------------------------- 1 | package com.hdfs; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.ByteArrayOutputStream; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.io.InputStreamReader; 9 | import java.io.OutputStreamWriter; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | import org.apache.hadoop.conf.Configuration; 14 | import org.apache.hadoop.fs.FSDataOutputStream; 15 | import org.apache.hadoop.fs.FileStatus; 16 | import org.apache.hadoop.fs.FileSystem; 17 | import org.apache.hadoop.fs.LocatedFileStatus; 18 | import org.apache.hadoop.fs.Path; 19 | import org.apache.hadoop.fs.RemoteIterator; 20 | import org.apache.hadoop.io.IOUtils; 21 | 22 | /** 23 | * Hadoop文件系统-工具类 24 | * @author 魏国兴 25 | */ 26 | public class HDFSHelper { 27 | 28 | /** 29 | * 判断文件或者目录是否存在 30 | * @param conf 31 | * @param path 32 | * @return 33 | * @throws IOException 34 | */ 35 | public static boolean exits(Configuration conf, String path) throws IOException { 36 | FileSystem fs = FileSystem.get(conf); 37 | return fs.exists(new Path(path)); 38 | } 39 | /** 40 | * 判断是否是目录 41 | * @param conf 42 | * @param path 43 | * @return 44 | * @throws IOException 45 | */ 46 | public static boolean isDirectory(Configuration conf, String path) throws IOException { 47 | FileSystem fs = FileSystem.get(conf); 48 | return fs.isDirectory(new Path(path)); 49 | } 50 | /** 51 | * 判断是否是文件 52 | * @param conf 53 | * @param path 54 | * @return 55 | * @throws IOException 56 | */ 57 | public static boolean isFile(Configuration conf, String path) throws IOException { 58 | FileSystem fs = FileSystem.get(conf); 59 | return fs.isFile(new Path(path)); 60 | } 61 | /** 62 | * 创建文件 63 | * 64 | * @param conf 65 | * @param filePath 66 | * @param contents 67 | * @throws IOException 68 | */ 69 | public static void createFile(Configuration conf, String filePath, byte[] contents) throws IOException { 70 | FileSystem fs = FileSystem.get(conf); 71 | Path path = new Path(filePath); 72 | FSDataOutputStream outputStream = fs.create(path); 73 | outputStream.write(contents); 74 | outputStream.close(); 75 | fs.close(); 76 | } 77 | 78 | /** 79 | * 创建文件 80 | * 81 | * @param conf 82 | * @param filePath 83 | * @param fileContent 84 | * @throws IOException 85 | */ 86 | public static void createFile(Configuration conf, String filePath, String fileContent) throws IOException { 87 | createFile(conf, filePath, fileContent.getBytes()); 88 | } 89 | 90 | /** 91 | * 上传文件 92 | * 93 | * @param conf 94 | * @param localFilePath 95 | * @param remoteFilePath 96 | * @throws IOException 97 | */ 98 | public static void uploadFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException { 99 | FileSystem fs = FileSystem.get(conf); 100 | Path localPath = new Path(localFilePath); 101 | Path remotePath = new Path(remoteFilePath); 102 | fs.copyFromLocalFile(true, true, localPath, remotePath); 103 | fs.close(); 104 | } 105 | /** 106 | * 下载文件 107 | * 108 | * @param conf 109 | * @param localFilePath 110 | * @param remoteFilePath 111 | * @throws IOException 112 | */ 113 | public static void dowloadFile(Configuration conf, String remoteFilePath,String localFilePath) throws IOException { 114 | FileSystem fs = FileSystem.get(conf); 115 | Path localPath = new Path(localFilePath); 116 | Path remotePath = new Path(remoteFilePath); 117 | fs.copyToLocalFile(false,remotePath,localPath,false); 118 | fs.close(); 119 | } 120 | /** 121 | * 删除目录或文件 122 | * 123 | * @param conf 124 | * @param remoteFilePath 125 | * @param recursive 126 | * @return 127 | * @throws IOException 128 | */ 129 | public static boolean deleteFile(Configuration conf, String remoteFilePath, boolean recursive) throws IOException { 130 | FileSystem fs = FileSystem.get(conf); 131 | boolean result = fs.delete(new Path(remoteFilePath), recursive); 132 | fs.close(); 133 | return result; 134 | } 135 | /** 136 | * 如果存在,即删除 137 | * 138 | * @param conf 139 | * @param remoteFilePath 140 | * @param recursive 141 | * @return 142 | * @throws IOException 143 | */ 144 | public static boolean deleteOnExit(Configuration conf, String remoteFilePath) throws IOException { 145 | FileSystem fs = FileSystem.get(conf); 146 | boolean result = fs.deleteOnExit(new Path(remoteFilePath)); 147 | fs.close(); 148 | return result; 149 | } 150 | /** 151 | * 删除目录或文件(如果有子目录,则级联删除) 152 | * 153 | * @param conf 154 | * @param remoteFilePath 155 | * @return 156 | * @throws IOException 157 | */ 158 | public static boolean deleteFile(Configuration conf, String remoteFilePath) throws IOException { 159 | return deleteFile(conf, remoteFilePath, true); 160 | } 161 | 162 | /** 163 | * 文件重命名 164 | * 165 | * @param conf 166 | * @param oldFileName 167 | * @param newFileName 168 | * @return 169 | * @throws IOException 170 | */ 171 | public static boolean renameFile(Configuration conf, String oldFileName, String newFileName) throws IOException { 172 | FileSystem fs = FileSystem.get(conf); 173 | Path oldPath = new Path(oldFileName); 174 | Path newPath = new Path(newFileName); 175 | boolean result = fs.rename(oldPath, newPath); 176 | fs.close(); 177 | return result; 178 | } 179 | 180 | /** 181 | * 创建目录 182 | * 183 | * @param conf 184 | * @param dirName 185 | * @return 186 | * @throws IOException 187 | */ 188 | public static boolean createDirectory(Configuration conf, String dirName) throws IOException { 189 | FileSystem fs = FileSystem.get(conf); 190 | Path dir = new Path(dirName); 191 | boolean result = fs.mkdirs(dir); 192 | fs.close(); 193 | return result; 194 | } 195 | 196 | /** 197 | * 列出指定路径下的所有文件(不包含目录) 198 | * 199 | * @param conf 200 | * @param basePath 201 | * @param recursive 202 | */ 203 | public static RemoteIterator listFiles(FileSystem fs, String basePath, boolean recursive) throws IOException { 204 | RemoteIterator fileStatusRemoteIterator = fs.listFiles(new Path(basePath), recursive); 205 | return fileStatusRemoteIterator; 206 | } 207 | 208 | /** 209 | * 列出指定路径下的文件（非递归） 210 | * 211 | * @param conf 212 | * @param basePath 213 | * @return 214 | * @throws IOException 215 | */ 216 | public static RemoteIterator listFiles(Configuration conf, String basePath) throws IOException { 217 | FileSystem fs = FileSystem.get(conf); 218 | RemoteIterator remoteIterator = fs.listFiles(new Path(basePath), false); 219 | fs.close(); 220 | return remoteIterator; 221 | } 222 | 223 | /** 224 | * 列出指定目录下的文件\子目录信息（非递归） 225 | * 226 | * @param conf 227 | * @param dirPath 228 | * @return 229 | * @throws IOException 230 | */ 231 | public static FileStatus[] listStatus(Configuration conf, String dirPath) throws IOException { 232 | FileSystem fs = FileSystem.get(conf); 233 | FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath)); 234 | fs.close(); 235 | return fileStatuses; 236 | } 237 | /** 238 | * 合并到 239 | * @param conf 240 | * @param src 241 | * @param output 242 | */ 243 | public static void mergeFiles(Configuration conf, String src, String output) { 244 | try { 245 | FileSystem fs = FileSystem.get(conf); 246 | fs.deleteOnExit(new Path(output)); 247 | 248 | BufferedWriter w = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(output)))); 249 | 250 | FileStatus[] outputFiles = fs.listStatus(new Path(src)); 251 | for (FileStatus fileStatus : outputFiles) { 252 | BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fileStatus.getPath()))); 253 | String line; 254 | while((line = reader.readLine()) != null) { 255 | w.append(line); 256 | w.newLine(); 257 | } 258 | reader.close(); 259 | } 260 | w.close(); 261 | } catch (Exception e) { 262 | throw new RuntimeException("Error merging files from dir: " + src + " into file: " + output, e); 263 | } 264 | } 265 | 266 | /** 267 | * 读取文件内容 268 | * 269 | * @param conf 270 | * @param filePath 271 | * @return 272 | * @throws IOException 273 | */ 274 | public static String readFile(Configuration conf, String filePath) throws IOException { 275 | String fileContent = null; 276 | FileSystem fs = FileSystem.get(conf); 277 | Path path = new Path(filePath); 278 | InputStream inputStream = null; 279 | ByteArrayOutputStream outputStream = null; 280 | try { 281 | inputStream = fs.open(path); 282 | outputStream = new ByteArrayOutputStream(inputStream.available()); 283 | IOUtils.copyBytes(inputStream, outputStream, conf); 284 | fileContent = outputStream.toString(); 285 | } finally { 286 | IOUtils.closeStream(inputStream); 287 | IOUtils.closeStream(outputStream); 288 | fs.close(); 289 | } 290 | return fileContent; 291 | } 292 | 293 | public static List readLines(Configuration conf, String filePath) throws IOException { 294 | return HDFSHelper.readLines(conf, filePath, null); 295 | } 296 | /** 297 | * 读取文件内容 298 | * 299 | * @param conf 300 | * @param filePath 301 | * @return 302 | * @throws IOException 303 | */ 304 | public static List readLines(Configuration conf, String filePath,Integer num) throws IOException { 305 | 306 | List lines = new ArrayList(); 307 | FileSystem fs = FileSystem.get(conf); 308 | Path path = new Path(filePath); 309 | 310 | BufferedReader reader = null; 311 | try { 312 | reader = new BufferedReader(new InputStreamReader(fs.open(path))); 313 | String line; 314 | Integer counter=0; 315 | while ((line = reader.readLine()) != null) { 316 | lines.add(line); 317 | counter++; 318 | if(num!=null&&counter>=num){ 319 | break; 320 | } 321 | } 322 | } catch (IOException e) { 323 | throw new RuntimeException("Error loading table in memory: " + path, e); 324 | } finally { 325 | try { 326 | if (reader != null) { 327 | reader.close(); 328 | } 329 | } catch (IOException e) { 330 | } 331 | } 332 | return lines; 333 | } 334 | public static void main(String args[]) throws IOException { 335 | 336 | Configuration conf = new Configuration(); 337 | 338 | String newDir = "/test"; 339 | //01.检测路径是否存在 测试 340 | if (HDFSHelper.exits(conf, newDir)) { 341 | System.out.println(newDir + " 已存在!"); 342 | } else { 343 | //02.创建目录测试 344 | boolean result = HDFSHelper.createDirectory(conf, newDir); 345 | if (result) { 346 | System.out.println(newDir + " 创建成功!"); 347 | } else { 348 | System.out.println(newDir + " 创建失败!"); 349 | } 350 | } 351 | String fileContent = "Hi,hadoop. I love you"; 352 | String newFileName = newDir + "/myfile.txt"; 353 | 354 | //03.创建文件测试 355 | HDFSHelper.createFile(conf, newFileName, fileContent); 356 | System.out.println(newFileName + " 创建成功"); 357 | 358 | //04.读取文件内容 测试 359 | System.out.println(newFileName + " 的内容为:\n" + HDFSHelper.readFile(conf, newFileName)); 360 | 361 | //05. 测试获取所有目录信息 362 | FileStatus[] dirs = HDFSHelper.listStatus(conf, "/"); 363 | System.out.println("--根目录下的所有子目录---"); 364 | for (FileStatus s : dirs) { 365 | System.out.println(s); 366 | } 367 | 368 | //06. 测试获取所有文件 369 | FileSystem fs = FileSystem.get(conf); 370 | RemoteIterator files = HDFSHelper.listFiles(fs, "/", true); 371 | System.out.println("--根目录下的所有文件---"); 372 | while (files.hasNext()) { 373 | System.out.println(files.next()); 374 | } 375 | fs.close(); 376 | 377 | //删除文件测试 378 | boolean isDeleted = HDFSHelper.deleteFile(conf, newDir); 379 | if(isDeleted){ 380 | System.out.println(newDir + " 已被删除"); 381 | } 382 | 383 | } 384 | } 385 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/mr/SortMapper.java: -------------------------------------------------------------------------------- 1 | package com.mr; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.Text; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | /** 8 | * 排序Mapper 9 | * @author 魏国兴 10 | * 11 | */ 12 | public class SortMapper extends Mapper { 13 | public void map(Object key,Text value,Context context) throws IOException, InterruptedException { 14 | context.write(new Text("sort"),new Text(value)); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/mr/SortReducer.java: -------------------------------------------------------------------------------- 1 | package com.mr; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.hadoop.io.NullWritable; 8 | import org.apache.hadoop.io.Text; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | 11 | import com.conf.SqlConf; 12 | import com.file.Table; 13 | import com.sql.SqlExeEngine; 14 | import com.sql.SqlParse; 15 | 16 | /** 17 | * 排序Reducer 18 | * 19 | * @author 魏国兴 20 | */ 21 | public class SortReducer extends Reducer { 22 | 23 | // SQL解析器 24 | private SqlParse sqlParse = null; 25 | 26 | // 反序列化生成table 27 | private String serialize = null; 28 | 29 | public void setup(Context context) throws IOException, InterruptedException { 30 | // sql对象 31 | String sql = context.getConfiguration().get(SqlConf.CONF_SQL); 32 | sqlParse = new SqlParse(sql); 33 | 34 | // main表 35 | if (sqlParse.get(SqlParse.JOIN) != null) { 36 | serialize = context.getConfiguration() 37 | .get(SqlConf.CONF_JOINTABLE); 38 | } else { 39 | serialize = context.getConfiguration().get(sqlParse.get(SqlParse.MAIN_TABLE)); 40 | } 41 | 42 | super.setup(context); 43 | } 44 | 45 | public void reduce(Text key, Iterable values, Context context) 46 | throws IOException, InterruptedException { 47 | 48 | // 初始化表 49 | Table table = initTable(values); 50 | 51 | // 构建SQL引擎 52 | SqlExeEngine sqlEngine = new SqlExeEngine(table); 53 | 54 | String distinct = sqlParse.get(SqlParse.DISTINCT); 55 | if (distinct != null) { 56 | sqlEngine.distinct(distinct); 57 | } 58 | // 执行order by 59 | String order = sqlParse.get(SqlParse.ORDER); 60 | if (order != null) { 61 | sqlEngine.order(order); 62 | } 63 | // 执行limit 64 | String limit = sqlParse.get(SqlParse.LIMIT); 65 | if (limit != null) { 66 | sqlEngine.limit(limit); 67 | } 68 | 69 | writeTable(context, sqlEngine.getTable()); 70 | } 71 | 72 | private List getRows(Iterable values) { 73 | List rows = new ArrayList(); 74 | for (Text t : values) { 75 | rows.add(t.toString()); 76 | } 77 | return rows; 78 | } 79 | 80 | /** 81 | * 将Reduce中的values转化成Table 82 | * 83 | * @param values 84 | * @return 85 | */ 86 | private Table initTable(Iterable values) { 87 | 88 | // 反序列化生成table 89 | Table table = new Table().diserialize(serialize); 90 | // 分隔符 91 | String split = table.getSplit(); 92 | table = new Table().diserialize(serialize); 93 | table.setFilter(null); 94 | String format = sqlParse.get("#mr.sort.format"); 95 | if(!format.equals("*")){ 96 | table.setFormat(format.replace(",", split)); 97 | } 98 | table.setRows(getRows(values)); 99 | 100 | return table; 101 | } 102 | 103 | /** 104 | * 将表格的内容写入到HDFS中 105 | * 106 | * @param context 107 | * @param table 108 | */ 109 | private void writeTable(Context context, Table table) throws IOException, 110 | InterruptedException { 111 | for (String row : table.getRows()) { 112 | context.write(NullWritable.get(), new Text(row)); 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/mr/SqlCombiner.java: -------------------------------------------------------------------------------- 1 | package com.mr; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | 10 | import com.conf.SqlConf; 11 | import com.file.Table; 12 | import com.sql.SqlExeEngine; 13 | import com.sql.SqlParse; 14 | 15 | /** 16 | * 执行SQL的Reducer函数 17 | * 18 | * @author 魏国兴 19 | */ 20 | public class SqlCombiner extends Reducer { 21 | 22 | // SQL解析器 23 | private SqlParse sqlParse = null; 24 | 25 | // 反序列化生成table 26 | private String serialize = null; 27 | 28 | public void setup(Context context) throws IOException, InterruptedException { 29 | // sql对象 30 | String sql = context.getConfiguration().get(SqlConf.CONF_SQL); 31 | sqlParse = new SqlParse(sql); 32 | 33 | // main表 34 | if (sqlParse.get(SqlParse.JOIN) != null) { 35 | serialize = context.getConfiguration() 36 | .get(SqlConf.CONF_JOINTABLE); 37 | } else { 38 | serialize = context.getConfiguration().get(sqlParse.get(SqlParse.MAIN_TABLE)); 39 | } 40 | } 41 | 42 | public void reduce(Text key, Iterable values, Context context) 43 | throws IOException, InterruptedException { 44 | 45 | // 初始化表 46 | Table table = initTable(values); 47 | 48 | // 构建SQL引擎 49 | SqlExeEngine sqlEngine = new SqlExeEngine(table); 50 | // 执行聚合操作 51 | String matrix = sqlParse.get(SqlParse.MATRIX); 52 | String group = sqlParse.get(SqlParse.GROUP); 53 | if (matrix != null) { 54 | sqlEngine.combine(matrix, group); 55 | } 56 | 57 | 58 | // 将table中的内容写入到hdfs中 59 | this.writeTable(key, context, sqlEngine.getTable()); 60 | } 61 | 62 | /** 63 | * 将Reduce中的values转化成Table 64 | * 65 | * @param values 66 | * @return 67 | */ 68 | private Table initTable(Iterable values) { 69 | 70 | // 反序列化生成table 71 | Table table = new Table().diserialize(serialize); 72 | table.setFilter(null); 73 | List rows = new ArrayList(); 74 | for (Text t : values) { 75 | rows.add(t.toString()); 76 | } 77 | table.setRows(rows); 78 | 79 | return table; 80 | } 81 | 82 | /** 83 | * 将表格的内容写入到HDFS中 84 | * 85 | * @param context 86 | * @param table 87 | */ 88 | private void writeTable(Text key, Context context, Table table) 89 | throws IOException, InterruptedException { 90 | for (String row : table.getRows()) { 91 | context.write(key, new Text(row)); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/mr/SqlMapper.java: -------------------------------------------------------------------------------- 1 | package com.mr; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | 10 | import com.conf.SqlConf; 11 | import com.file.HDFSTable; 12 | import com.file.Table; 13 | import com.sql.SqlExeEngine; 14 | import com.sql.SqlParse; 15 | 16 | /** 17 | * 执行SQL的Mapper函数 18 | * @author 魏国兴 19 | */ 20 | public class SqlMapper extends Mapper { 21 | 22 | // main主表 23 | private Table table=null; 24 | // join表 25 | private Map joins=new HashMap(); 26 | // SQL解析器 27 | private SqlParse sqlParse=null; 28 | 29 | public void setup(Context context) throws IOException, InterruptedException { 30 | 31 | // sql对象 32 | String sql=context.getConfiguration().get(SqlConf.CONF_SQL); 33 | sqlParse=new SqlParse(sql); 34 | 35 | // main表 36 | String json=context.getConfiguration().get(sqlParse.get(SqlParse.MAIN_TABLE)); 37 | this.table=new Table().diserialize(json); 38 | 39 | 40 | // join表 41 | String join = sqlParse.get(SqlParse.JOIN); 42 | if (join != null) { 43 | String joins[]=sqlParse.get(SqlParse.JOIN_TABLE).split(","); 44 | for(String t : joins){ 45 | json=context.getConfiguration().get(t); 46 | Table table=new HDFSTable(context.getConfiguration(), new Table().diserialize(json)); 47 | this.joins.put(table.getName(),table); 48 | } 49 | } 50 | super.setup(context); 51 | } 52 | 53 | public void map(Object key,Text value,Context context) throws IOException, InterruptedException { 54 | // 添加当前行并执行过滤 55 | boolean isEmpty=table.setRow(value.toString()).filterRows().isEmpty(); 56 | if(isEmpty) return; 57 | 58 | // 构建SQL引擎 59 | SqlExeEngine sqlEngine=new SqlExeEngine(table); 60 | 61 | // 连接Join表 62 | String join = sqlParse.get(SqlParse.JOIN); 63 | if (join != null) { 64 | for (String en : join.split("\\|")) { 65 | String table = en.split("on")[0]; 66 | String on = en.split("on")[1]; 67 | String name = SqlParse.getTable(table); 68 | sqlEngine.join(joins.get(name), on); 69 | } 70 | } 71 | // 执行where 72 | String where = sqlParse.get(SqlParse.WHERE); 73 | if (where != null) { 74 | sqlEngine.where(where); 75 | } 76 | 77 | // 执行分组 78 | String group = sqlParse.get(SqlParse.GROUP); 79 | for(String row:sqlEngine.getTable().getRows()){ 80 | String ky="id";// 默认按照id分组 81 | if(group!=null){ 82 | ky = sqlEngine.getTable().getColumns(row, group); 83 | } 84 | context.write(new Text(ky),new Text(row)); 85 | } 86 | 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/mr/SqlReducer.java: -------------------------------------------------------------------------------- 1 | package com.mr; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.hadoop.io.NullWritable; 8 | import org.apache.hadoop.io.Text; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | 11 | import com.conf.SqlConf; 12 | import com.file.Table; 13 | import com.sql.SqlExeEngine; 14 | import com.sql.SqlParse; 15 | 16 | /** 17 | * 执行SQL的Reducer函数 18 | * 19 | * @author 魏国兴 20 | */ 21 | public class SqlReducer extends Reducer { 22 | 23 | // SQL解析器 24 | private SqlParse sqlParse = null; 25 | 26 | // 反序列化生成table 27 | private String serialize = null; 28 | 29 | public void setup(Context context) throws IOException, InterruptedException { 30 | // sql对象 31 | String sql = context.getConfiguration().get(SqlConf.CONF_SQL); 32 | sqlParse = new SqlParse(sql); 33 | 34 | // main表 35 | if (sqlParse.get(SqlParse.JOIN) != null) { 36 | serialize = context.getConfiguration() 37 | .get(SqlConf.CONF_JOINTABLE); 38 | } else { 39 | serialize = context.getConfiguration().get(sqlParse.get(SqlParse.MAIN_TABLE)); 40 | } 41 | } 42 | 43 | public void reduce(Text key, Iterable values, Context context) 44 | throws IOException, InterruptedException { 45 | 46 | // 初始化表 47 | Table table = initTable(values); 48 | 49 | // 构建SQL引擎 50 | SqlExeEngine sqlEngine = new SqlExeEngine(table); 51 | 52 | // 执行聚合操作 53 | String matrix = sqlParse.get("#mr.reduce.matrix"); 54 | String group = sqlParse.get(SqlParse.GROUP); 55 | if (matrix != null) { 56 | sqlEngine.group(matrix, group); 57 | } 58 | // 执行过滤 59 | String select = sqlParse.get("#mr.reduce.select"); 60 | if (select != null) { 61 | sqlEngine.select(select); 62 | } 63 | 64 | // 将table中的内容写入到hdfs中 65 | this.writeTable(context, sqlEngine.getTable()); 66 | } 67 | 68 | /** 69 | * 将Reduce中的values转化成Table 70 | * 71 | * @param values 72 | * @return 73 | */ 74 | private Table initTable(Iterable values) { 75 | 76 | // 反序列化生成table 77 | Table table = new Table().diserialize(serialize); 78 | table.setFilter(null); 79 | // 分隔符 80 | String split = table.getSplit(); 81 | 82 | // Mapper输入格式：t.id|t.name|count(t.id) as t.count 83 | // 目标格式：t.id|t.name|c#t.id 84 | // 更具matrix和group计算新的table格式 85 | String group = sqlParse.get(SqlParse.GROUP); 86 | String format =sqlParse.get("#mr.reduce.format").replace(",", split); 87 | if(group!=null){ 88 | format = group.replace(",", split) + split 89 | + sqlParse.get("#mr.reduce.format").replace(",", split); 90 | } 91 | table.setFormat(format); 92 | 93 | List rows = new ArrayList(); 94 | for (Text t : values) { 95 | rows.add(t.toString()); 96 | } 97 | table.setRows(rows); 98 | 99 | return table; 100 | } 101 | 102 | /** 103 | * 将表格的内容写入到HDFS中 104 | * 105 | * @param context 106 | * @param table 107 | */ 108 | private void writeTable(Context context, Table table) throws IOException, 109 | InterruptedException { 110 | for (String row : table.getRows()) { 111 | context.write(NullWritable.get(), new Text(row)); 112 | } 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/sql/SqlExeEngine.java: -------------------------------------------------------------------------------- 1 | package com.sql; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Comparator; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.Stack; 9 | 10 | import com.file.Table; 11 | 12 | /** 13 | * SQL执行单元引擎 14 | * 15 | * @author 魏国兴 16 | * 17 | */ 18 | public class SqlExeEngine { 19 | 20 | // 操作表 21 | private Table table = null; 22 | 23 | public SqlExeEngine() { 24 | } 25 | 26 | public SqlExeEngine(Table table) { 27 | this.table = table; 28 | } 29 | 30 | public SqlExeEngine join(Table join, String on) { 31 | return this.join(join, on, false); 32 | } 33 | 34 | public SqlExeEngine join(Table join, String on, boolean leftJoin) { 35 | 36 | // 链接条件 37 | class On { 38 | public String mainCol;// 主表字段 39 | public String joinCol;// join表字段 40 | 41 | public On(String mainCol, String joinCol) { 42 | this.mainCol = mainCol; 43 | this.joinCol = joinCol; 44 | } 45 | } 46 | 47 | // 表连接的主表 48 | Table main = this.table; 49 | 50 | String[] splits = on.trim().split("and"); 51 | 52 | List ons = new ArrayList(); 53 | 54 | for (String wherecase : splits) { 55 | String[] joincases = wherecase.replaceAll("\\s+", "").split("="); 56 | 57 | String mainCol;// 主表字段 58 | String joinCol;// join表字段 59 | if (joincases[0].indexOf(this.table.getName()) > -1) { 60 | mainCol = joincases[0].substring(joincases[0].indexOf(".") + 1); 61 | joinCol = joincases[1].substring(joincases[1].indexOf(".") + 1); 62 | } else { 63 | joinCol = joincases[0].substring(joincases[0].indexOf(".") + 1); 64 | mainCol = joincases[1].substring(joincases[1].indexOf(".") + 1); 65 | } 66 | ons.add(new On(mainCol, joinCol)); 67 | } 68 | 69 | List combineRow = new ArrayList(); 70 | // 拼接主表行和join表的行 71 | for (String row : main.getRows()) { 72 | 73 | Map condition = new HashMap(); 74 | for (On o : ons) { 75 | String column = main.getColumns(row, o.mainCol); 76 | condition.put(o.joinCol, column); 77 | } 78 | boolean leftFlag = true; 79 | for (String joinRow : join.getRows()) { 80 | boolean flag = true; 81 | for (Map.Entry entry : condition.entrySet()) { 82 | String col = entry.getKey(); 83 | String val = entry.getValue(); 84 | if (!join.getColumns(joinRow, col).equals(val)) { 85 | flag = false; 86 | } 87 | } 88 | if (flag) { 89 | leftFlag = false; 90 | combineRow.add(row + this.table.getSplit() + joinRow); 91 | } 92 | } 93 | if (leftJoin && leftFlag) { 94 | combineRow.add(row); 95 | } 96 | 97 | } 98 | 99 | // 合并后的表格格式 100 | String name1 = main.getName(); 101 | String name2 = join.getName(); 102 | String format1 = main.getFormat(); 103 | String format2 = join.getFormat(); 104 | 105 | String split = this.table.getSplit(); 106 | format1 = (name1 + "." + format1).replace(split, split + name1 + "."); 107 | format2 = (name2 + "." + format2).replace(split, split + name2 + "."); 108 | 109 | String format = format1 + split + format2; 110 | 111 | this.table = new Table(name1 + "," + name2, null, 112 | this.table.getSplit(), format, null,combineRow); 113 | 114 | return this; 115 | } 116 | 117 | public SqlExeEngine where(String where) { 118 | // 以空白字符切分where语句 119 | String arry[] = where.split("\\s+"); 120 | // 存放操作符 121 | Stack opt = new Stack(); 122 | // 存放后缀表达式 123 | List output = new ArrayList(); 124 | 125 | // 将中缀表达式转换为后缀表达式 126 | for (int i = 0; i < arry.length; i++) { 127 | String it = arry[i]; 128 | if (it.equals("and") || it.equals("or")) { 129 | if (opt.isEmpty() || opt.peek().equals("(")) { 130 | opt.push(it); 131 | } else { 132 | output.add(opt.pop()); 133 | } 134 | } else if (it.equals("(")) { 135 | opt.push(it); 136 | } else if (it.equals(")")) { 137 | String el = opt.pop(); 138 | while (el.equals("(") == false) { 139 | output.add(el); 140 | el = opt.pop(); 141 | } 142 | } else { 143 | output.add(it); 144 | } 145 | } 146 | while (!opt.isEmpty()) { 147 | output.add(opt.pop()); 148 | } 149 | 150 | List filters = new ArrayList(); 151 | // 过滤table中的列 152 | for (String row : table.getRows()) { 153 | // 存放操作符 154 | Stack opts = new Stack(); 155 | // 解析后缀表达式并运算结果 156 | for (String v : output) { 157 | if (v.equals("or")) { 158 | boolean v1 = opts.pop(); 159 | boolean v2 = opts.pop(); 160 | opts.push(v1 || v2); 161 | } else if (v.equals("and")) { 162 | boolean v1 = opts.pop(); 163 | boolean v2 = opts.pop(); 164 | opts.push(v1 && v2); 165 | } else { 166 | opts.push(this.filter(table, row, v)); 167 | } 168 | } 169 | // 取出计算结果 170 | boolean res = opts.pop(); 171 | if (res) { 172 | filters.add(row); 173 | } 174 | } 175 | 176 | // 过滤后的列 177 | table.setRows(filters); 178 | 179 | return this; 180 | } 181 | /** 182 | * 判断某行知否满足wherecase这个条件 183 | * @param t 184 | * @param row 185 | * @param wherecase 186 | * @return 187 | */ 188 | private boolean filter(Table t, String row, String wherecase) { 189 | 190 | if (wherecase.equals("true") || wherecase.equals("false")) { 191 | return Boolean.parseBoolean(wherecase); 192 | } 193 | 194 | String splits[] = wherecase.split(">=|<=|=|!=|<|>|like"); 195 | String col = splits[0]; 196 | String val = splits[1]; 197 | String opt = wherecase.substring(col.length(), 198 | wherecase.length() - val.length()); 199 | 200 | String v = t.getColumns(row, col); 201 | 202 | boolean res = false; 203 | if (v.matches("\\d+\\.?\\d+")) { 204 | Double v1 = Double.parseDouble(v); 205 | Double v2 = Double.parseDouble(val); 206 | if (opt.equals(">=")) { 207 | res = v1.compareTo(v2) >= 0; 208 | } else if (opt.equals("<=")) { 209 | res = v1.compareTo(v2) <= 0; 210 | } else if (opt.equals("=")) { 211 | res = v.equals(val); 212 | } else if (opt.equals("!=")) { 213 | res = !v.equals(val); 214 | } else if (opt.equals("<")) { 215 | res = v1.compareTo(v2) < 0; 216 | } else if (opt.equals(">")) { 217 | res = v1.compareTo(v2) > 0; 218 | } else if (opt.equals("like")) { 219 | res = v.matches(val); 220 | } 221 | }else{ 222 | if (opt.equals(">=")) { 223 | res = v.compareTo(val) >= 0; 224 | } else if (opt.equals("<=")) { 225 | res = v.compareTo(val) <= 0; 226 | } else if (opt.equals("=")) { 227 | res = v.equals(val); 228 | } else if (opt.equals("!=")) { 229 | res = !v.equals(val); 230 | } else if (opt.equals("<")) { 231 | res = v.compareTo(val) < 0; 232 | } else if (opt.equals(">")) { 233 | res = v.compareTo(val) > 0; 234 | } else if (opt.equals("like")) { 235 | res = v.matches(val); 236 | } 237 | } 238 | return res; 239 | 240 | } 241 | /** 242 | * 返回指定行数 243 | * 244 | * @param limit 245 | * @return 246 | */ 247 | public SqlExeEngine limit(String limit) { 248 | 249 | String splits[] = limit.split(","); 250 | 251 | int start = Integer.parseInt(splits[0]); 252 | int end = Integer.parseInt(splits[1]); 253 | 254 | List rows = new ArrayList(); 255 | for (int i = start; i < end; i++) { 256 | if(i>=table.getRows().size()){ 257 | break; 258 | } 259 | rows.add(table.getRows().get(i)); 260 | } 261 | table.setRows(rows); 262 | 263 | return this; 264 | } 265 | 266 | /** 267 | * 对所有行进行排序 268 | * 269 | * @param order 270 | * @return 271 | */ 272 | public SqlExeEngine order(String order) { 273 | 274 | // 获取要排序的字段 275 | String splits[] = order.split(","); 276 | table.getRows().sort(new Comparator() { 277 | @Override 278 | public int compare(String row1, String row2) { 279 | int res = 0; 280 | for (String cols : splits) { 281 | // 排序的字段和排序的方式 282 | String col = cols.split("\\s+")[0]; 283 | String type = cols.split("\\s+")[1]; 284 | String val1 = table.getColumns(row1, col); 285 | String val2 = table.getColumns(row2, col); 286 | // 数字 287 | if (val1.matches("\\d+\\.?\\d+")) { 288 | Integer v1 = Integer.parseInt(val1); 289 | Integer v2 = Integer.parseInt(val2); 290 | if (type.toLowerCase().equals("asc")) { 291 | res = v1.compareTo(v2); 292 | } else { 293 | res = v2.compareTo(v1); 294 | } 295 | // 字符串 296 | } else { 297 | if (type.toLowerCase().equals("asc")) { 298 | res = val1.compareTo(val2); 299 | } else { 300 | res = val2.compareTo(val1); 301 | } 302 | } 303 | } 304 | return res; 305 | } 306 | }); 307 | 308 | return this; 309 | } 310 | 311 | /** 312 | * 返回select的列 313 | * 314 | * @param select 315 | * @return 316 | */ 317 | public SqlExeEngine select(String select) { 318 | 319 | // 处理select*的情况 320 | String split = this.table.getSplit(); 321 | if (select.equals("*")) { 322 | select = this.table.getFormat().replace(split, ","); 323 | } 324 | 325 | String format = ""; 326 | List rows = new ArrayList(); 327 | 328 | // 选取select中的列 329 | String splits[] = select.split(","); 330 | for (String r : table.getRows()) { 331 | String row = ""; 332 | format = ""; 333 | 334 | for (String col : splits) { 335 | if (col.matches(".*(sum|avg|max|min|count).*")) { 336 | col = SqlParse.getMetrix(col, "alias"); 337 | } 338 | if (col.contains("distinct")) { 339 | col = SqlParse.getDistinct(col); 340 | } 341 | format += col + split; 342 | row += table.getColumns(r, col) + split; 343 | } 344 | 345 | format = format.substring(0, format.length() - 1); 346 | row = row.substring(0, row.length() - 1); 347 | rows.add(row); 348 | } 349 | table.setFormat(format); 350 | table.setRows(rows); 351 | 352 | return this; 353 | } 354 | 355 | public SqlExeEngine group(String matrix, String group) { 356 | return this.group(matrix, group, null); 357 | } 358 | 359 | public SqlExeEngine combine(String matrix, String group) { 360 | return this.group(matrix, group, "#"); 361 | } 362 | 363 | /** 364 | * group操作 365 | * 366 | * @param matrix 367 | * @param group 368 | * @param sp 369 | * @return 370 | */ 371 | public SqlExeEngine group(String matrix, String group, String combine) { 372 | 373 | // table分隔符 374 | String split = this.table.getSplit(); 375 | 376 | // group上下文 377 | Map> context = new HashMap>(); 378 | 379 | if (group == null) { 380 | context.put("nogroup", this.getTable().getRows());// group 381 | // 为null时,就是按id分组 382 | } else { 383 | // mapper 384 | for (String row : this.table.getRows()) { 385 | String key = table.getColumns(row, group); 386 | 387 | List rows = context.get(key); 388 | rows = rows == null ? new ArrayList() : rows; 389 | 390 | rows.add(row); 391 | context.put(key, rows); 392 | } 393 | } 394 | 395 | // reducer 396 | Map collects = new HashMap(); 397 | String format = ""; 398 | 399 | for (Map.Entry> entry : context.entrySet()) { 400 | 401 | // 分组的key 402 | String key = entry.getKey(); 403 | List values = entry.getValue(); 404 | 405 | format = ""; 406 | String aggregate = ""; 407 | 408 | // 需要聚合的指标 409 | String[] matrixs = matrix.split(","); 410 | for (String mtrix : matrixs) { 411 | 412 | String func = SqlParse.getMetrix(mtrix, "func"); 413 | String field = SqlParse.getMetrix(mtrix, "field"); 414 | String alias = SqlParse.getMetrix(mtrix, "alias"); 415 | 416 | format += alias + split; 417 | 418 | if (func.equals("sum")) { 419 | aggregate += this.sum(field, values) + split; 420 | } 421 | if (func.equals("avg")) { 422 | if (combine != null) { 423 | aggregate += this.sum(field, values) + combine 424 | + this.count(field, values) + split; 425 | } else { 426 | aggregate += this.avg(field, values) + split; 427 | } 428 | } 429 | if (func.equals("max")) { 430 | aggregate += this.max(field, values) + split; 431 | } 432 | if (func.equals("min")) { 433 | aggregate += this.min(field, values) + split; 434 | } 435 | if (func.equals("count")) { 436 | if (combine != null) { 437 | aggregate += combine + this.count(field, values) 438 | + split; 439 | } else { 440 | aggregate += this.count(field, values) + split; 441 | } 442 | } 443 | 444 | } 445 | 446 | format = format.substring(0, format.length() - 1); 447 | aggregate = aggregate.substring(0, aggregate.length() - 1); 448 | 449 | collects.put(key, aggregate); 450 | } 451 | 452 | List rows = new ArrayList(); 453 | for (Map.Entry entry : collects.entrySet()) { 454 | String key = entry.getKey(); 455 | if (key.equals("nogroup")) { 456 | rows.add(entry.getValue()); 457 | } else { 458 | rows.add(key + split + entry.getValue()); 459 | } 460 | } 461 | if (group != null) { 462 | format = group.replace(",", split) + split + format; 463 | } 464 | this.table.setFormat(format); 465 | this.table.setRows(rows); 466 | 467 | return this; 468 | } 469 | 470 | public SqlExeEngine distinct(String distinct) { 471 | 472 | distinct = SqlParse.getDistinct(distinct); 473 | 474 | Map map = new HashMap(); 475 | for (String row : this.table.getRows()) { 476 | map.put(this.table.getColumns(row, distinct), row); 477 | } 478 | this.table.setRows(new ArrayList(map.values())); 479 | return this; 480 | } 481 | 482 | public String count(String field) { 483 | return this.count(field, this.table.getRows()); 484 | } 485 | 486 | /** 487 | * 求指定列的行数 488 | * 489 | * @param field 490 | * @param rows 491 | * @return 492 | */ 493 | public String count(String field, List rows) { 494 | Double count = new Double(0); 495 | if(field.equals("*")){ 496 | count+=rows.size(); 497 | }else{ 498 | for (String row : rows) { 499 | String col = this.table.getColumns(row, field); 500 | if (col.startsWith("#")) { 501 | count += Double.parseDouble(col.substring(1)); 502 | } else { 503 | count += 1; 504 | } 505 | } 506 | } 507 | return Db2Str(count); 508 | } 509 | 510 | public String max(String field) { 511 | return this.max(field, this.table.getRows()); 512 | } 513 | 514 | /** 515 | * 求指定列的最大值 516 | * 517 | * @param field 518 | * @param rows 519 | * @return 520 | */ 521 | public String max(String field, List rows) { 522 | Double max = Double.MIN_VALUE; 523 | for (String row : rows) { 524 | Double value = this.table.getColumn(row, field, Double.class); 525 | if (value > max) { 526 | max = value; 527 | } 528 | } 529 | return Db2Str(max); 530 | } 531 | 532 | public String min(String field) { 533 | return this.min(field, this.table.getRows()); 534 | } 535 | 536 | /** 537 | * 求指定列的最小值 538 | * 539 | * @param field 540 | * @param rows 541 | * @return 542 | */ 543 | public String min(String field, List rows) { 544 | Double min = Double.MAX_VALUE; 545 | for (String row : rows) { 546 | Double value = this.table.getColumn(row, field, Double.class); 547 | if (value < min) { 548 | min = value; 549 | } 550 | } 551 | return Db2Str(min); 552 | } 553 | 554 | public String avg(String field) { 555 | return this.avg(field, this.table.getRows()); 556 | } 557 | 558 | /** 559 | * 求指定列列的平均数 560 | * 561 | * @param field 562 | * @param rows 563 | * @return 564 | */ 565 | public String avg(String field, List rows) { 566 | Double sum = new Double(0); 567 | Double count = new Double(0); 568 | Double avg = new Double(0); 569 | for (String row : rows) { 570 | String col = this.table.getColumns(row, field); 571 | if (col.contains("#")) { 572 | sum += Double.parseDouble(col.split("#")[0]); 573 | count += Double.parseDouble(col.split("#")[1]); 574 | } else { 575 | sum += Double.parseDouble(col); 576 | count += 1; 577 | } 578 | } 579 | avg = sum / count; 580 | return Db2Str(avg); 581 | } 582 | 583 | public String sum(String field) { 584 | return this.sum(field, this.table.getRows()); 585 | } 586 | 587 | /** 588 | * 计算row中指定field列的和 589 | * 590 | * @param field 591 | * @param rows 592 | * @return 593 | */ 594 | public String sum(String field, List rows) { 595 | Double sum = new Double(0); 596 | for (String row : rows) { 597 | sum += this.table.getColumn(row, field, Double.class); 598 | } 599 | return Db2Str(sum); 600 | } 601 | 602 | 603 | private String Db2Str(Double res) { 604 | return res.toString().matches("\\d+.0+") ? res.intValue() + "" : res 605 | .toString(); 606 | } 607 | 608 | public String toString() { 609 | return this.table.toString(); 610 | } 611 | 612 | public Table getTable() { 613 | return table; 614 | } 615 | 616 | public void setTable(Table table) { 617 | this.table = table; 618 | } 619 | 620 | } 621 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/com/sql/SqlParse.java: -------------------------------------------------------------------------------- 1 | package com.sql; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * SQL解析器 10 | * 11 | * @author 魏国兴 12 | * 13 | */ 14 | public class SqlParse { 15 | 16 | // 输入输出路径 17 | public static final String INPUT="sql.input"; 18 | public static final String OUTPUT="sql.output"; 19 | 20 | // 主表和Join表 21 | public static final String MAIN_TABLE="#main.table"; 22 | public static final String JOIN_TABLE="#join.table"; 23 | 24 | // SQL元语 25 | public static final String CREATE="create"; 26 | public static final String SELECT="select"; 27 | public static final String FROM="from"; 28 | public static final String JOIN="join"; 29 | public static final String WHERE="where"; 30 | public static final String GROUP="group by"; 31 | public static final String ORDER="order by"; 32 | public static final String LIMIT="limit"; 33 | 34 | public static final String MATRIX="matrix"; 35 | public static final String DISTINCT="distinct"; 36 | 37 | 38 | // MapReduce使用 39 | public static final String MR_REDUCE_FORMAT="#mr.reduce.format"; 40 | public static final String MR_REDUCE_MATRIX="#mr.reduce.matrix"; 41 | public static final String MR_REDUCE_SELECT="#mr.reduce.select"; 42 | 43 | public static final String MR_SORT_FORMAT="#mr.sort.format"; 44 | 45 | /** 46 | * 存放SQL解析的结果 47 | */ 48 | private Map SQL = new HashMap(); 49 | 50 | /** 51 | * 解析SQL 52 | * 53 | * @param sql 54 | */ 55 | public SqlParse(String sql) { 56 | 57 | // 格式化SQL 58 | sql = sql 59 | .trim() 60 | .replaceAll("\\s*(create|select|from|join|where|group by|order by|limit)\\s+","}$0{") 61 | .substring(1) 62 | .concat("}") 63 | .replaceAll("\\s*(,|>|<|=|!=|>=|<=|like)\\s*", "$1"); 64 | 65 | // 拆分SQL 66 | Pattern p = Pattern 67 | .compile("(create|select|from|join|where|group by|order by|limit)\\s+\\{(.*?)\\}"); 68 | Matcher m = p.matcher(sql); 69 | 70 | // 将正则匹配的结果存放到Map中 71 | while (m.find()) { 72 | String func = m.group(1); 73 | String param = m.group(2); 74 | 75 | // 处理多个join的时候 76 | SQL.put(func, SQL.get(func) == null ? param : SQL.get(func) + "|" 77 | + param); 78 | } 79 | 80 | // 从SQL中解析出 matrix和distinct 81 | String select=SQL.get(SELECT); 82 | String[] splits = select.split(","); 83 | 84 | String matrix=""; 85 | String distinct=null; 86 | 87 | for (String field : splits) { 88 | if (field.matches(".*(sum|avg|max|min|count).*")) { 89 | matrix +=field+","; 90 | } else if (field.matches(".*distinct.*")) { 91 | distinct = field; 92 | } 93 | } 94 | 95 | if (!matrix.equals("")) { 96 | matrix = matrix.substring(0,matrix.length()-1); 97 | }else{ 98 | matrix=null; 99 | } 100 | SQL.put(MATRIX,matrix); 101 | SQL.put(DISTINCT,distinct); 102 | 103 | 104 | // 格式化where语句 105 | String where=SQL.get(WHERE); 106 | if(where!=null){ 107 | where = where.replace("&&", "and").replace("||", "or") 108 | .replaceAll("\\(|\\)", " $0 "); 109 | 110 | String[] compare = { ">=", "<=", "=", "!=", "<", ">", "like" }; 111 | for (int i = 0; i < compare.length; i++) { 112 | where = where.replaceAll("\\s*" + compare[i] + "\\s*", compare[i]); 113 | } 114 | SQL.put(WHERE,where); 115 | } 116 | 117 | // 主表和join表 118 | SQL.put(MAIN_TABLE, this.getMainTable()); 119 | SQL.put(JOIN_TABLE, this.getJoinTables()); 120 | SQL.put(OUTPUT,this.getOutPath()); 121 | SQL.put(INPUT,this.getInputPath()); 122 | 123 | // 用于MapReduce中 124 | String mtxReg="([s,a,m,n,c])(um|vg|ax|in|ount)\\s*\\((.*?)\\)\\s+as\\s+([\\w|\\.]+)"; 125 | if(matrix!=null){ 126 | SQL.put(MR_REDUCE_FORMAT,matrix.replaceAll(mtxReg,"$1#$3")); 127 | SQL.put(MR_REDUCE_MATRIX,matrix.replaceAll(mtxReg,"$1$2($1#$3) as $4")); 128 | } 129 | SQL.put(MR_REDUCE_SELECT,select.replaceAll(mtxReg,"$1$2($1#$3) as $4")); 130 | SQL.put(MR_SORT_FORMAT,select.replaceAll(mtxReg,"$4")); 131 | 132 | 133 | } 134 | /** 135 | * 获取SQL中的value 136 | * 137 | * @param key 138 | * @return 139 | */ 140 | public String get(String key) { 141 | return SQL.get(key); 142 | } 143 | /** 144 | * 获取SQL中的value 145 | * 146 | * @param key 147 | * @return 148 | */ 149 | public String get(String key,String def) { 150 | String val=SQL.get(key); 151 | return val==null?def:val; 152 | } 153 | 154 | /** 155 | * 解析distinct字段 156 | * @param distinct 157 | * @return 158 | */ 159 | public static String getDistinct(String distinct){ 160 | String regx="distinct\\s+\\((.*?)\\)"; 161 | Pattern p = Pattern.compile(regx); 162 | Matcher m = p.matcher(distinct); 163 | String col = null; 164 | if (m.find()) { 165 | col = m.group(1); 166 | } 167 | 168 | return col; 169 | } 170 | /** 171 | * 解析Select中的聚合函数 172 | * @param mtrix 173 | * @param flag 174 | * @return 175 | */ 176 | public static String getMetrix(String mtrix,String flag){ 177 | 178 | if (!mtrix.contains("as")) { 179 | mtrix += " as matrix_name"; 180 | } 181 | Pattern p = Pattern 182 | .compile("(sum|avg|max|min|count)\\s*\\((.*?)\\)\\s+as\\s+([\\w|\\.]+)"); 183 | Matcher m = p.matcher(mtrix); 184 | 185 | String func = null; 186 | String field = null; 187 | String alias = null; 188 | if (m.find()) { 189 | func = m.group(1); 190 | field = m.group(2); 191 | alias = m.group(3).trim(); 192 | if (alias.equals("matrix_name")) { 193 | alias = func + "(" + field + ")"; 194 | } 195 | } 196 | 197 | Map res=new HashMap(); 198 | res.put("func",func); 199 | res.put("field",field); 200 | res.put("alias",alias); 201 | 202 | return res.get(flag); 203 | } 204 | 205 | 206 | /** 207 | * 从table串中解析表名 208 | * 209 | * @param table eg:classpath:student.txt s 210 | * @return 211 | */ 212 | public static String getTable(String table) { 213 | String[] splits = table.trim().split("\\s+"); 214 | if (splits.length == 1) { 215 | return splits[0]; 216 | } else { 217 | return splits[1]; 218 | } 219 | } 220 | 221 | /** 222 | * 从table串中解析表路径 223 | * 224 | * @param table eg:classpath:student.txt s 225 | * @return 226 | */ 227 | public static String getPath(String table) { 228 | String[] splits = table.trim().split("\\s+"); 229 | if (splits.length == 1) { 230 | return null; 231 | } else { 232 | return splits[0]; 233 | } 234 | } 235 | 236 | 237 | /** 238 | * 返回需要join的表的表名 239 | * 240 | * @return 241 | */ 242 | private String getJoinTables(){ 243 | String joins=""; 244 | String join = this.get("join"); 245 | if (join != null) { 246 | for (String en : join.split("\\|")) { 247 | String table = en.split("on")[0]; 248 | String name = SqlParse.getTable(table); 249 | joins=name+","; 250 | } 251 | } 252 | if(!joins.equals("")){ 253 | joins=joins.substring(0,joins.length()-1); 254 | } 255 | return joins; 256 | } 257 | /** 258 | * 返回主表名称 259 | * 260 | * @return 261 | */ 262 | private String getMainTable(){ 263 | String from = this.get("from"); 264 | return SqlParse.getTable(from); 265 | } 266 | 267 | /** 268 | * 返回输出目录 269 | * @return 270 | */ 271 | private String getOutPath(){ 272 | String create =this.get("create"); 273 | if(create!=null){ 274 | create=create.split("\\s+")[0]; 275 | } 276 | return create; 277 | } 278 | /** 279 | * 返回输入目录 280 | * eg:input:name,input:name 281 | * @return 282 | */ 283 | private String getInputPath(){ 284 | 285 | String input=null; 286 | 287 | String regex="(\\S+)\\s*(as)?\\s*(\\S*)"; 288 | String replace="$1:$3"; 289 | 290 | String from=this.get("from"); 291 | String joins=this.get("join"); 292 | 293 | if(from!=null){ 294 | String tmp=from.trim().replaceAll(regex,replace); 295 | input=(tmp.endsWith(":")?tmp.substring(0,tmp.length()-1):tmp)+","; 296 | } 297 | if (joins != null) { 298 | for (String join : joins.split("\\|")) { 299 | String table = join.split("on")[0]; 300 | String tmp=table.trim().replaceAll(regex,replace); 301 | input+=(tmp.endsWith(":")?tmp.substring(0,tmp.length()-1):tmp)+","; 302 | } 303 | } 304 | if(input!=null) input=input.substring(0, input.length()-1); 305 | return input; 306 | } 307 | } 308 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.io.nativeio; 19 | 20 | import java.io.File; 21 | import java.io.FileDescriptor; 22 | import java.io.FileInputStream; 23 | import java.io.FileOutputStream; 24 | import java.io.IOException; 25 | import java.io.RandomAccessFile; 26 | import java.lang.reflect.Field; 27 | import java.nio.ByteBuffer; 28 | import java.nio.MappedByteBuffer; 29 | import java.nio.channels.FileChannel; 30 | import java.util.Map; 31 | import java.util.concurrent.ConcurrentHashMap; 32 | 33 | import org.apache.hadoop.classification.InterfaceAudience; 34 | import org.apache.hadoop.classification.InterfaceStability; 35 | import org.apache.hadoop.conf.Configuration; 36 | import org.apache.hadoop.fs.CommonConfigurationKeys; 37 | import org.apache.hadoop.fs.HardLink; 38 | import org.apache.hadoop.io.IOUtils; 39 | import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException; 40 | import org.apache.hadoop.util.NativeCodeLoader; 41 | import org.apache.hadoop.util.Shell; 42 | import org.apache.hadoop.util.PerformanceAdvisory; 43 | import org.apache.commons.logging.Log; 44 | import org.apache.commons.logging.LogFactory; 45 | 46 | import sun.misc.Unsafe; 47 | 48 | import com.google.common.annotations.VisibleForTesting; 49 | 50 | /** 51 | * JNI wrappers for various native IO-related calls not available in Java. 52 | * These functions should generally be used alongside a fallback to another 53 | * more portable mechanism. 54 | */ 55 | @InterfaceAudience.Private 56 | @InterfaceStability.Unstable 57 | public class NativeIO { 58 | public static class POSIX { 59 | // Flags for open() call from bits/fcntl.h 60 | public static final int O_RDONLY = 00; 61 | public static final int O_WRONLY = 01; 62 | public static final int O_RDWR = 02; 63 | public static final int O_CREAT = 0100; 64 | public static final int O_EXCL = 0200; 65 | public static final int O_NOCTTY = 0400; 66 | public static final int O_TRUNC = 01000; 67 | public static final int O_APPEND = 02000; 68 | public static final int O_NONBLOCK = 04000; 69 | public static final int O_SYNC = 010000; 70 | public static final int O_ASYNC = 020000; 71 | public static final int O_FSYNC = O_SYNC; 72 | public static final int O_NDELAY = O_NONBLOCK; 73 | 74 | // Flags for posix_fadvise() from bits/fcntl.h 75 | /* No further special treatment. */ 76 | public static final int POSIX_FADV_NORMAL = 0; 77 | /* Expect random page references. */ 78 | public static final int POSIX_FADV_RANDOM = 1; 79 | /* Expect sequential page references. */ 80 | public static final int POSIX_FADV_SEQUENTIAL = 2; 81 | /* Will need these pages. */ 82 | public static final int POSIX_FADV_WILLNEED = 3; 83 | /* Don't need these pages. */ 84 | public static final int POSIX_FADV_DONTNEED = 4; 85 | /* Data will be accessed once. */ 86 | public static final int POSIX_FADV_NOREUSE = 5; 87 | 88 | 89 | /* Wait upon writeout of all pages 90 | in the range before performing the 91 | write. */ 92 | public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1; 93 | /* Initiate writeout of all those 94 | dirty pages in the range which are 95 | not presently under writeback. */ 96 | public static final int SYNC_FILE_RANGE_WRITE = 2; 97 | 98 | /* Wait upon writeout of all pages in 99 | the range after performing the 100 | write. */ 101 | public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4; 102 | 103 | private static final Log LOG = LogFactory.getLog(NativeIO.class); 104 | 105 | private static boolean nativeLoaded = false; 106 | private static boolean fadvisePossible = true; 107 | private static boolean syncFileRangePossible = true; 108 | 109 | static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = 110 | "hadoop.workaround.non.threadsafe.getpwuid"; 111 | static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = true; 112 | 113 | private static long cacheTimeout = -1; 114 | 115 | private static CacheManipulator cacheManipulator = new CacheManipulator(); 116 | 117 | public static CacheManipulator getCacheManipulator() { 118 | return cacheManipulator; 119 | } 120 | 121 | public static void setCacheManipulator(CacheManipulator cacheManipulator) { 122 | POSIX.cacheManipulator = cacheManipulator; 123 | } 124 | 125 | /** 126 | * Used to manipulate the operating system cache. 127 | */ 128 | @VisibleForTesting 129 | public static class CacheManipulator { 130 | public void mlock(String identifier, ByteBuffer buffer, 131 | long len) throws IOException { 132 | POSIX.mlock(buffer, len); 133 | } 134 | 135 | public long getMemlockLimit() { 136 | return NativeIO.getMemlockLimit(); 137 | } 138 | 139 | public long getOperatingSystemPageSize() { 140 | return NativeIO.getOperatingSystemPageSize(); 141 | } 142 | 143 | public void posixFadviseIfPossible(String identifier, 144 | FileDescriptor fd, long offset, long len, int flags) 145 | throws NativeIOException { 146 | NativeIO.POSIX.posixFadviseIfPossible(identifier, fd, offset, 147 | len, flags); 148 | } 149 | 150 | public boolean verifyCanMlock() { 151 | return NativeIO.isAvailable(); 152 | } 153 | } 154 | 155 | /** 156 | * A CacheManipulator used for testing which does not actually call mlock. 157 | * This allows many tests to be run even when the operating system does not 158 | * allow mlock, or only allows limited mlocking. 159 | */ 160 | @VisibleForTesting 161 | public static class NoMlockCacheManipulator extends CacheManipulator { 162 | public void mlock(String identifier, ByteBuffer buffer, 163 | long len) throws IOException { 164 | LOG.info("mlocking " + identifier); 165 | } 166 | 167 | public long getMemlockLimit() { 168 | return 1125899906842624L; 169 | } 170 | 171 | public long getOperatingSystemPageSize() { 172 | return 4096; 173 | } 174 | 175 | public boolean verifyCanMlock() { 176 | return true; 177 | } 178 | } 179 | 180 | static { 181 | if (NativeCodeLoader.isNativeCodeLoaded()) { 182 | try { 183 | Configuration conf = new Configuration(); 184 | workaroundNonThreadSafePasswdCalls = conf.getBoolean( 185 | WORKAROUND_NON_THREADSAFE_CALLS_KEY, 186 | WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT); 187 | 188 | initNative(); 189 | nativeLoaded = true; 190 | 191 | cacheTimeout = conf.getLong( 192 | CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY, 193 | CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * 194 | 1000; 195 | LOG.debug("Initialized cache for IDs to User/Group mapping with a " + 196 | " cache timeout of " + cacheTimeout/1000 + " seconds."); 197 | 198 | } catch (Throwable t) { 199 | // This can happen if the user has an older version of libhadoop.so 200 | // installed - in this case we can continue without native IO 201 | // after warning 202 | PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t); 203 | } 204 | } 205 | } 206 | 207 | /** 208 | * Return true if the JNI-based native IO extensions are available. 209 | */ 210 | public static boolean isAvailable() { 211 | return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; 212 | } 213 | 214 | private static void assertCodeLoaded() throws IOException { 215 | if (!isAvailable()) { 216 | throw new IOException("NativeIO was not loaded"); 217 | } 218 | } 219 | 220 | /** Wrapper around open(2) */ 221 | public static native FileDescriptor open(String path, int flags, int mode) throws IOException; 222 | /** Wrapper around fstat(2) */ 223 | private static native Stat fstat(FileDescriptor fd) throws IOException; 224 | 225 | /** Native chmod implementation. On UNIX, it is a wrapper around chmod(2) */ 226 | private static native void chmodImpl(String path, int mode) throws IOException; 227 | 228 | public static void chmod(String path, int mode) throws IOException { 229 | if (!Shell.WINDOWS) { 230 | chmodImpl(path, mode); 231 | } else { 232 | try { 233 | chmodImpl(path, mode); 234 | } catch (NativeIOException nioe) { 235 | if (nioe.getErrorCode() == 3) { 236 | throw new NativeIOException("No such file or directory", 237 | Errno.ENOENT); 238 | } else { 239 | LOG.warn(String.format("NativeIO.chmod error (%d): %s", 240 | nioe.getErrorCode(), nioe.getMessage())); 241 | throw new NativeIOException("Unknown error", Errno.UNKNOWN); 242 | } 243 | } 244 | } 245 | } 246 | 247 | /** Wrapper around posix_fadvise(2) */ 248 | static native void posix_fadvise( 249 | FileDescriptor fd, long offset, long len, int flags) throws NativeIOException; 250 | 251 | /** Wrapper around sync_file_range(2) */ 252 | static native void sync_file_range( 253 | FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException; 254 | 255 | /** 256 | * Call posix_fadvise on the given file descriptor. See the manpage 257 | * for this syscall for more information. On systems where this 258 | * call is not available, does nothing. 259 | * 260 | * @throws NativeIOException if there is an error with the syscall 261 | */ 262 | static void posixFadviseIfPossible(String identifier, 263 | FileDescriptor fd, long offset, long len, int flags) 264 | throws NativeIOException { 265 | if (nativeLoaded && fadvisePossible) { 266 | try { 267 | posix_fadvise(fd, offset, len, flags); 268 | } catch (UnsupportedOperationException uoe) { 269 | fadvisePossible = false; 270 | } catch (UnsatisfiedLinkError ule) { 271 | fadvisePossible = false; 272 | } 273 | } 274 | } 275 | 276 | /** 277 | * Call sync_file_range on the given file descriptor. See the manpage 278 | * for this syscall for more information. On systems where this 279 | * call is not available, does nothing. 280 | * 281 | * @throws NativeIOException if there is an error with the syscall 282 | */ 283 | public static void syncFileRangeIfPossible( 284 | FileDescriptor fd, long offset, long nbytes, int flags) 285 | throws NativeIOException { 286 | if (nativeLoaded && syncFileRangePossible) { 287 | try { 288 | sync_file_range(fd, offset, nbytes, flags); 289 | } catch (UnsupportedOperationException uoe) { 290 | syncFileRangePossible = false; 291 | } catch (UnsatisfiedLinkError ule) { 292 | syncFileRangePossible = false; 293 | } 294 | } 295 | } 296 | 297 | static native void mlock_native( 298 | ByteBuffer buffer, long len) throws NativeIOException; 299 | 300 | /** 301 | * Locks the provided direct ByteBuffer into memory, preventing it from 302 | * swapping out. After a buffer is locked, future accesses will not incur 303 | * a page fault. 304 | * 305 | * See the mlock(2) man page for more information. 306 | * 307 | * @throws NativeIOException 308 | */ 309 | static void mlock(ByteBuffer buffer, long len) 310 | throws IOException { 311 | assertCodeLoaded(); 312 | if (!buffer.isDirect()) { 313 | throw new IOException("Cannot mlock a non-direct ByteBuffer"); 314 | } 315 | mlock_native(buffer, len); 316 | } 317 | 318 | /** 319 | * Unmaps the block from memory. See munmap(2). 320 | * 321 | * There isn't any portable way to unmap a memory region in Java. 322 | * So we use the sun.nio method here. 323 | * Note that unmapping a memory region could cause crashes if code 324 | * continues to reference the unmapped code. However, if we don't 325 | * manually unmap the memory, we are dependent on the finalizer to 326 | * do it, and we have no idea when the finalizer will run. 327 | * 328 | * @param buffer The buffer to unmap. 329 | */ 330 | public static void munmap(MappedByteBuffer buffer) { 331 | if (buffer instanceof sun.nio.ch.DirectBuffer) { 332 | sun.misc.Cleaner cleaner = 333 | ((sun.nio.ch.DirectBuffer)buffer).cleaner(); 334 | cleaner.clean(); 335 | } 336 | } 337 | 338 | /** Linux only methods used for getOwner() implementation */ 339 | private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException; 340 | private static native String getUserName(long uid) throws IOException; 341 | 342 | /** 343 | * Result type of the fstat call 344 | */ 345 | public static class Stat { 346 | private int ownerId, groupId; 347 | private String owner, group; 348 | private int mode; 349 | 350 | // Mode constants 351 | public static final int S_IFMT = 0170000; /* type of file */ 352 | public static final int S_IFIFO = 0010000; /* named pipe (fifo) */ 353 | public static final int S_IFCHR = 0020000; /* character special */ 354 | public static final int S_IFDIR = 0040000; /* directory */ 355 | public static final int S_IFBLK = 0060000; /* block special */ 356 | public static final int S_IFREG = 0100000; /* regular */ 357 | public static final int S_IFLNK = 0120000; /* symbolic link */ 358 | public static final int S_IFSOCK = 0140000; /* socket */ 359 | public static final int S_IFWHT = 0160000; /* whiteout */ 360 | public static final int S_ISUID = 0004000; /* set user id on execution */ 361 | public static final int S_ISGID = 0002000; /* set group id on execution */ 362 | public static final int S_ISVTX = 0001000; /* save swapped text even after use */ 363 | public static final int S_IRUSR = 0000400; /* read permission, owner */ 364 | public static final int S_IWUSR = 0000200; /* write permission, owner */ 365 | public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ 366 | 367 | Stat(int ownerId, int groupId, int mode) { 368 | this.ownerId = ownerId; 369 | this.groupId = groupId; 370 | this.mode = mode; 371 | } 372 | 373 | Stat(String owner, String group, int mode) { 374 | if (!Shell.WINDOWS) { 375 | this.owner = owner; 376 | } else { 377 | this.owner = stripDomain(owner); 378 | } 379 | if (!Shell.WINDOWS) { 380 | this.group = group; 381 | } else { 382 | this.group = stripDomain(group); 383 | } 384 | this.mode = mode; 385 | } 386 | 387 | @Override 388 | public String toString() { 389 | return "Stat(owner='" + owner + "', group='" + group + "'" + 390 | ", mode=" + mode + ")"; 391 | } 392 | 393 | public String getOwner() { 394 | return owner; 395 | } 396 | public String getGroup() { 397 | return group; 398 | } 399 | public int getMode() { 400 | return mode; 401 | } 402 | } 403 | 404 | /** 405 | * Returns the file stat for a file descriptor. 406 | * 407 | * @param fd file descriptor. 408 | * @return the file descriptor file stat. 409 | * @throws IOException thrown if there was an IO error while obtaining the file stat. 410 | */ 411 | public static Stat getFstat(FileDescriptor fd) throws IOException { 412 | Stat stat = null; 413 | if (!Shell.WINDOWS) { 414 | stat = fstat(fd); 415 | stat.owner = getName(IdCache.USER, stat.ownerId); 416 | stat.group = getName(IdCache.GROUP, stat.groupId); 417 | } else { 418 | try { 419 | stat = fstat(fd); 420 | } catch (NativeIOException nioe) { 421 | if (nioe.getErrorCode() == 6) { 422 | throw new NativeIOException("The handle is invalid.", 423 | Errno.EBADF); 424 | } else { 425 | LOG.warn(String.format("NativeIO.getFstat error (%d): %s", 426 | nioe.getErrorCode(), nioe.getMessage())); 427 | throw new NativeIOException("Unknown error", Errno.UNKNOWN); 428 | } 429 | } 430 | } 431 | return stat; 432 | } 433 | 434 | private static String getName(IdCache domain, int id) throws IOException { 435 | Map idNameCache = (domain == IdCache.USER) 436 | ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE; 437 | String name; 438 | CachedName cachedName = idNameCache.get(id); 439 | long now = System.currentTimeMillis(); 440 | if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) { 441 | name = cachedName.name; 442 | } else { 443 | name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id); 444 | if (LOG.isDebugEnabled()) { 445 | String type = (domain == IdCache.USER) ? "UserName" : "GroupName"; 446 | LOG.debug("Got " + type + " " + name + " for ID " + id + 447 | " from the native implementation"); 448 | } 449 | cachedName = new CachedName(name, now); 450 | idNameCache.put(id, cachedName); 451 | } 452 | return name; 453 | } 454 | 455 | static native String getUserName(int uid) throws IOException; 456 | static native String getGroupName(int uid) throws IOException; 457 | 458 | private static class CachedName { 459 | final long timestamp; 460 | final String name; 461 | 462 | public CachedName(String name, long timestamp) { 463 | this.name = name; 464 | this.timestamp = timestamp; 465 | } 466 | } 467 | 468 | private static final Map USER_ID_NAME_CACHE = 469 | new ConcurrentHashMap(); 470 | 471 | private static final Map GROUP_ID_NAME_CACHE = 472 | new ConcurrentHashMap(); 473 | 474 | private enum IdCache { USER, GROUP } 475 | 476 | public final static int MMAP_PROT_READ = 0x1; 477 | public final static int MMAP_PROT_WRITE = 0x2; 478 | public final static int MMAP_PROT_EXEC = 0x4; 479 | 480 | public static native long mmap(FileDescriptor fd, int prot, 481 | boolean shared, long length) throws IOException; 482 | 483 | public static native void munmap(long addr, long length) 484 | throws IOException; 485 | } 486 | 487 | private static boolean workaroundNonThreadSafePasswdCalls = false; 488 | 489 | 490 | public static class Windows { 491 | // Flags for CreateFile() call on Windows 492 | public static final long GENERIC_READ = 0x80000000L; 493 | public static final long GENERIC_WRITE = 0x40000000L; 494 | 495 | public static final long FILE_SHARE_READ = 0x00000001L; 496 | public static final long FILE_SHARE_WRITE = 0x00000002L; 497 | public static final long FILE_SHARE_DELETE = 0x00000004L; 498 | 499 | public static final long CREATE_NEW = 1; 500 | public static final long CREATE_ALWAYS = 2; 501 | public static final long OPEN_EXISTING = 3; 502 | public static final long OPEN_ALWAYS = 4; 503 | public static final long TRUNCATE_EXISTING = 5; 504 | 505 | public static final long FILE_BEGIN = 0; 506 | public static final long FILE_CURRENT = 1; 507 | public static final long FILE_END = 2; 508 | 509 | public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L; 510 | 511 | /** Wrapper around CreateFile() on Windows */ 512 | public static native FileDescriptor createFile(String path, 513 | long desiredAccess, long shareMode, long creationDisposition) 514 | throws IOException; 515 | 516 | /** Wrapper around SetFilePointer() on Windows */ 517 | public static native long setFilePointer(FileDescriptor fd, 518 | long distanceToMove, long moveMethod) throws IOException; 519 | 520 | /** Windows only methods used for getOwner() implementation */ 521 | private static native String getOwner(FileDescriptor fd) throws IOException; 522 | 523 | /** Supported list of Windows access right flags */ 524 | public static enum AccessRight { 525 | ACCESS_READ (0x0001), // FILE_READ_DATA 526 | ACCESS_WRITE (0x0002), // FILE_WRITE_DATA 527 | ACCESS_EXECUTE (0x0020); // FILE_EXECUTE 528 | 529 | private final int accessRight; 530 | AccessRight(int access) { 531 | accessRight = access; 532 | } 533 | 534 | public int accessRight() { 535 | return accessRight; 536 | } 537 | }; 538 | 539 | /** Windows only method used to check if the current process has requested 540 | * access rights on the given path. */ 541 | private static native boolean access0(String path, int requestedAccess); 542 | 543 | /** 544 | * Checks whether the current process has desired access rights on 545 | * the given path. 546 | * 547 | * Longer term this native function can be substituted with JDK7 548 | * function Files#isReadable, isWritable, isExecutable. 549 | * 550 | * @param path input path 551 | * @param desiredAccess ACCESS_READ, ACCESS_WRITE or ACCESS_EXECUTE 552 | * @return true if access is allowed 553 | * @throws IOException I/O exception on error 554 | */ 555 | public static boolean access(String path, AccessRight desiredAccess) 556 | throws IOException { 557 | return true; 558 | //return access0(path, desiredAccess.accessRight()); 559 | } 560 | 561 | /** 562 | * Extends both the minimum and maximum working set size of the current 563 | * process. This method gets the current minimum and maximum working set 564 | * size, adds the requested amount to each and then sets the minimum and 565 | * maximum working set size to the new values. Controlling the working set 566 | * size of the process also controls the amount of memory it can lock. 567 | * 568 | * @param delta amount to increment minimum and maximum working set size 569 | * @throws IOException for any error 570 | * @see POSIX#mlock(ByteBuffer, long) 571 | */ 572 | public static native void extendWorkingSetSize(long delta) throws IOException; 573 | 574 | static { 575 | if (NativeCodeLoader.isNativeCodeLoaded()) { 576 | try { 577 | initNative(); 578 | nativeLoaded = true; 579 | } catch (Throwable t) { 580 | // This can happen if the user has an older version of libhadoop.so 581 | // installed - in this case we can continue without native IO 582 | // after warning 583 | PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t); 584 | } 585 | } 586 | } 587 | } 588 | 589 | private static final Log LOG = LogFactory.getLog(NativeIO.class); 590 | 591 | private static boolean nativeLoaded = false; 592 | 593 | static { 594 | if (NativeCodeLoader.isNativeCodeLoaded()) { 595 | try { 596 | initNative(); 597 | nativeLoaded = true; 598 | } catch (Throwable t) { 599 | // This can happen if the user has an older version of libhadoop.so 600 | // installed - in this case we can continue without native IO 601 | // after warning 602 | PerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t); 603 | } 604 | } 605 | } 606 | 607 | /** 608 | * Return true if the JNI-based native IO extensions are available. 609 | */ 610 | public static boolean isAvailable() { 611 | return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; 612 | } 613 | 614 | /** Initialize the JNI method ID and class ID cache */ 615 | private static native void initNative(); 616 | 617 | /** 618 | * Get the maximum number of bytes that can be locked into memory at any 619 | * given point. 620 | * 621 | * @return 0 if no bytes can be locked into memory; 622 | * Long.MAX_VALUE if there is no limit; 623 | * The number of bytes that can be locked into memory otherwise. 624 | */ 625 | static long getMemlockLimit() { 626 | return isAvailable() ? getMemlockLimit0() : 0; 627 | } 628 | 629 | private static native long getMemlockLimit0(); 630 | 631 | /** 632 | * @return the operating system's page size. 633 | */ 634 | static long getOperatingSystemPageSize() { 635 | try { 636 | Field f = Unsafe.class.getDeclaredField("theUnsafe"); 637 | f.setAccessible(true); 638 | Unsafe unsafe = (Unsafe)f.get(null); 639 | return unsafe.pageSize(); 640 | } catch (Throwable e) { 641 | LOG.warn("Unable to get operating system page size. Guessing 4096.", e); 642 | return 4096; 643 | } 644 | } 645 | 646 | private static class CachedUid { 647 | final long timestamp; 648 | final String username; 649 | public CachedUid(String username, long timestamp) { 650 | this.timestamp = timestamp; 651 | this.username = username; 652 | } 653 | } 654 | private static final Map uidCache = 655 | new ConcurrentHashMap(); 656 | private static long cacheTimeout; 657 | private static boolean initialized = false; 658 | 659 | /** 660 | * The Windows logon name has two part, NetBIOS domain name and 661 | * user account name, of the format DOMAIN\UserName. This method 662 | * will remove the domain part of the full logon name. 663 | * 664 | * @param Fthe full principal name containing the domain 665 | * @return name with domain removed 666 | */ 667 | private static String stripDomain(String name) { 668 | int i = name.indexOf('\\'); 669 | if (i != -1) 670 | name = name.substring(i + 1); 671 | return name; 672 | } 673 | 674 | public static String getOwner(FileDescriptor fd) throws IOException { 675 | ensureInitialized(); 676 | if (Shell.WINDOWS) { 677 | String owner = Windows.getOwner(fd); 678 | owner = stripDomain(owner); 679 | return owner; 680 | } else { 681 | long uid = POSIX.getUIDforFDOwnerforOwner(fd); 682 | CachedUid cUid = uidCache.get(uid); 683 | long now = System.currentTimeMillis(); 684 | if (cUid != null && (cUid.timestamp + cacheTimeout) > now) { 685 | return cUid.username; 686 | } 687 | String user = POSIX.getUserName(uid); 688 | LOG.info("Got UserName " + user + " for UID " + uid 689 | + " from the native implementation"); 690 | cUid = new CachedUid(user, now); 691 | uidCache.put(uid, cUid); 692 | return user; 693 | } 694 | } 695 | 696 | /** 697 | * Create a FileInputStream that shares delete permission on the 698 | * file opened, i.e. other process can delete the file the 699 | * FileInputStream is reading. Only Windows implementation uses 700 | * the native interface. 701 | */ 702 | public static FileInputStream getShareDeleteFileInputStream(File f) 703 | throws IOException { 704 | if (!Shell.WINDOWS) { 705 | // On Linux the default FileInputStream shares delete permission 706 | // on the file opened. 707 | // 708 | return new FileInputStream(f); 709 | } else { 710 | // Use Windows native interface to create a FileInputStream that 711 | // shares delete permission on the file opened. 712 | // 713 | FileDescriptor fd = Windows.createFile( 714 | f.getAbsolutePath(), 715 | Windows.GENERIC_READ, 716 | Windows.FILE_SHARE_READ | 717 | Windows.FILE_SHARE_WRITE | 718 | Windows.FILE_SHARE_DELETE, 719 | Windows.OPEN_EXISTING); 720 | return new FileInputStream(fd); 721 | } 722 | } 723 | 724 | /** 725 | * Create a FileInputStream that shares delete permission on the 726 | * file opened at a given offset, i.e. other process can delete 727 | * the file the FileInputStream is reading. Only Windows implementation 728 | * uses the native interface. 729 | */ 730 | public static FileInputStream getShareDeleteFileInputStream(File f, long seekOffset) 731 | throws IOException { 732 | if (!Shell.WINDOWS) { 733 | @SuppressWarnings("resource") 734 | RandomAccessFile rf = new RandomAccessFile(f, "r"); 735 | if (seekOffset > 0) { 736 | rf.seek(seekOffset); 737 | } 738 | return new FileInputStream(rf.getFD()); 739 | } else { 740 | // Use Windows native interface to create a FileInputStream that 741 | // shares delete permission on the file opened, and set it to the 742 | // given offset. 743 | // 744 | FileDescriptor fd = NativeIO.Windows.createFile( 745 | f.getAbsolutePath(), 746 | NativeIO.Windows.GENERIC_READ, 747 | NativeIO.Windows.FILE_SHARE_READ | 748 | NativeIO.Windows.FILE_SHARE_WRITE | 749 | NativeIO.Windows.FILE_SHARE_DELETE, 750 | NativeIO.Windows.OPEN_EXISTING); 751 | if (seekOffset > 0) 752 | NativeIO.Windows.setFilePointer(fd, seekOffset, NativeIO.Windows.FILE_BEGIN); 753 | return new FileInputStream(fd); 754 | } 755 | } 756 | 757 | /** 758 | * Create the specified File for write access, ensuring that it does not exist. 759 | * @param f the file that we want to create 760 | * @param permissions we want to have on the file (if security is enabled) 761 | * 762 | * @throws AlreadyExistsException if the file already exists 763 | * @throws IOException if any other error occurred 764 | */ 765 | public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions) 766 | throws IOException { 767 | if (!Shell.WINDOWS) { 768 | // Use the native wrapper around open(2) 769 | try { 770 | FileDescriptor fd = NativeIO.POSIX.open(f.getAbsolutePath(), 771 | NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT 772 | | NativeIO.POSIX.O_EXCL, permissions); 773 | return new FileOutputStream(fd); 774 | } catch (NativeIOException nioe) { 775 | if (nioe.getErrno() == Errno.EEXIST) { 776 | throw new AlreadyExistsException(nioe); 777 | } 778 | throw nioe; 779 | } 780 | } else { 781 | // Use the Windows native APIs to create equivalent FileOutputStream 782 | try { 783 | FileDescriptor fd = NativeIO.Windows.createFile(f.getCanonicalPath(), 784 | NativeIO.Windows.GENERIC_WRITE, 785 | NativeIO.Windows.FILE_SHARE_DELETE 786 | | NativeIO.Windows.FILE_SHARE_READ 787 | | NativeIO.Windows.FILE_SHARE_WRITE, 788 | NativeIO.Windows.CREATE_NEW); 789 | NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions); 790 | return new FileOutputStream(fd); 791 | } catch (NativeIOException nioe) { 792 | if (nioe.getErrorCode() == 80) { 793 | // ERROR_FILE_EXISTS 794 | // 80 (0x50) 795 | // The file exists 796 | throw new AlreadyExistsException(nioe); 797 | } 798 | throw nioe; 799 | } 800 | } 801 | } 802 | 803 | private synchronized static void ensureInitialized() { 804 | if (!initialized) { 805 | cacheTimeout = 806 | new Configuration().getLong("hadoop.security.uid.cache.secs", 807 | 4*60*60) * 1000; 808 | LOG.info("Initialized cache for UID to User mapping with a cache" + 809 | " timeout of " + cacheTimeout/1000 + " seconds."); 810 | initialized = true; 811 | } 812 | } 813 | 814 | /** 815 | * A version of renameTo that throws a descriptive exception when it fails. 816 | * 817 | * @param src The source path 818 | * @param dst The destination path 819 | * 820 | * @throws NativeIOException On failure. 821 | */ 822 | public static void renameTo(File src, File dst) 823 | throws IOException { 824 | if (!nativeLoaded) { 825 | if (!src.renameTo(dst)) { 826 | throw new IOException("renameTo(src=" + src + ", dst=" + 827 | dst + ") failed."); 828 | } 829 | } else { 830 | renameTo0(src.getAbsolutePath(), dst.getAbsolutePath()); 831 | } 832 | } 833 | 834 | public static void link(File src, File dst) throws IOException { 835 | if (!nativeLoaded) { 836 | HardLink.createHardLink(src, dst); 837 | } else { 838 | link0(src.getAbsolutePath(), dst.getAbsolutePath()); 839 | } 840 | } 841 | 842 | /** 843 | * A version of renameTo that throws a descriptive exception when it fails. 844 | * 845 | * @param src The source path 846 | * @param dst The destination path 847 | * 848 | * @throws NativeIOException On failure. 849 | */ 850 | private static native void renameTo0(String src, String dst) 851 | throws NativeIOException; 852 | 853 | private static native void link0(String src, String dst) 854 | throws NativeIOException; 855 | 856 | /** 857 | * Unbuffered file copy from src to dst without tainting OS buffer cache 858 | * 859 | * In POSIX platform: 860 | * It uses FileChannel#transferTo() which internally attempts 861 | * unbuffered IO on OS with native sendfile64() support and falls back to 862 | * buffered IO otherwise. 863 | * 864 | * It minimizes the number of FileChannel#transferTo call by passing the the 865 | * src file size directly instead of a smaller size as the 3rd parameter. 866 | * This saves the number of sendfile64() system call when native sendfile64() 867 | * is supported. In the two fall back cases where sendfile is not supported, 868 | * FileChannle#transferTo already has its own batching of size 8 MB and 8 KB, 869 | * respectively. 870 | * 871 | * In Windows Platform: 872 | * It uses its own native wrapper of CopyFileEx with COPY_FILE_NO_BUFFERING 873 | * flag, which is supported on Windows Server 2008 and above. 874 | * 875 | * Ideally, we should use FileChannel#transferTo() across both POSIX and Windows 876 | * platform. Unfortunately, the wrapper(Java_sun_nio_ch_FileChannelImpl_transferTo0) 877 | * used by FileChannel#transferTo for unbuffered IO is not implemented on Windows. 878 | * Based on OpenJDK 6/7/8 source code, Java_sun_nio_ch_FileChannelImpl_transferTo0 879 | * on Windows simply returns IOS_UNSUPPORTED. 880 | * 881 | * Note: This simple native wrapper does minimal parameter checking before copy and 882 | * consistency check (e.g., size) after copy. 883 | * It is recommended to use wrapper function like 884 | * the Storage#nativeCopyFileUnbuffered() function in hadoop-hdfs with pre/post copy 885 | * checks. 886 | * 887 | * @param src The source path 888 | * @param dst The destination path 889 | * @throws IOException 890 | */ 891 | public static void copyFileUnbuffered(File src, File dst) throws IOException { 892 | if (nativeLoaded && Shell.WINDOWS) { 893 | copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath()); 894 | } else { 895 | FileInputStream fis = null; 896 | FileOutputStream fos = null; 897 | FileChannel input = null; 898 | FileChannel output = null; 899 | try { 900 | fis = new FileInputStream(src); 901 | fos = new FileOutputStream(dst); 902 | input = fis.getChannel(); 903 | output = fos.getChannel(); 904 | long remaining = input.size(); 905 | long position = 0; 906 | long transferred = 0; 907 | while (remaining > 0) { 908 | transferred = input.transferTo(position, remaining, output); 909 | remaining -= transferred; 910 | position += transferred; 911 | } 912 | } finally { 913 | IOUtils.cleanup(LOG, output); 914 | IOUtils.cleanup(LOG, fos); 915 | IOUtils.cleanup(LOG, input); 916 | IOUtils.cleanup(LOG, fis); 917 | } 918 | } 919 | } 920 | 921 | private static native void copyFileUnbuffered0(String src, String dst) 922 | throws NativeIOException; 923 | } 924 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/resources/chain.conf: -------------------------------------------------------------------------------- 1 | ############## 2 | # HDFS 3 | ############## 4 | log.hdfs=hdfs://192.168.1.32:9000 5 | 6 | ############## 7 | # SQL 8 | ############## 9 | 10 | # 简单查询 11 | log.sql1=create /sql/out1 as select * from s order by id desc 12 | 13 | # 简单查询 14 | log.sql2=create /sql/out2 as select id,name,grade from s where id>10 order by grade desc limit 0,10 15 | 16 | # 查询最高的成绩 17 | log.sql3=create /sql/out3 as select max(grade) as grade from s 18 | 19 | # 表连接 20 | log.sql4=create /sql/out4 as select s.id,s.name,s.grade,t.id,t.name from s join t on s.tid=t.id limit 0,10 21 | 22 | # 分组查询 23 | log.sql5=create /sql/out5 as select s.tid,count(s.id) as s.count from s group by s.tid 24 | 25 | # 表连接分组查询 26 | log.sql6=create /sql/out6 as select t.name,count(t.id) as t.count from s join t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,5 27 | 28 | # log chain 29 | log.chain=sql1,sql2,sql3,sql4,sql5,sql6 30 | 31 | ############## 32 | # VAR 33 | ############## 34 | # log table 35 | log.table.t=/sql/teacher.txt:id|name:#split:#filter 36 | log.table.s=/sql/student.txt:id|name|grade|tid:#split:#filter 37 | 38 | # split 39 | log.split=| 40 | 41 | # log filter 42 | log.filter=(^[^#].*) -------------------------------------------------------------------------------- /src/sql-egine/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Apache Software Foundation 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Define some default values that can be overridden by system properties 20 | hadoop.root.logger=INFO,console 21 | hadoop.log.dir=. 22 | hadoop.log.file=hadoop.log 23 | 24 | # Define the root logger to the system property "hadoop.root.logger". 25 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 26 | 27 | # Logging Threshold 28 | log4j.threshold=ALL 29 | 30 | # Null Appender 31 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 32 | 33 | # 34 | # Rolling File Appender - cap space usage at 5gb. 35 | # 36 | hadoop.log.maxfilesize=256MB 37 | hadoop.log.maxbackupindex=20 38 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 39 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 40 | 41 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 42 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 43 | 44 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 45 | 46 | # Pattern format: Date LogLevel LoggerName LogMessage 47 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 48 | # Debugging Pattern format 49 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 50 | 51 | 52 | # 53 | # Daily Rolling File Appender 54 | # 55 | 56 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 57 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 58 | 59 | # Rollver at midnight 60 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 61 | 62 | # 30-day backup 63 | #log4j.appender.DRFA.MaxBackupIndex=30 64 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 65 | 66 | # Pattern format: Date LogLevel LoggerName LogMessage 67 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 68 | # Debugging Pattern format 69 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 70 | 71 | 72 | # 73 | # console 74 | # Add "console" to rootlogger above if you want to use this 75 | # 76 | 77 | log4j.appender.console=org.apache.log4j.ConsoleAppender 78 | log4j.appender.console.target=System.err 79 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 80 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 81 | 82 | # 83 | # TaskLog Appender 84 | # 85 | 86 | #Default values 87 | hadoop.tasklog.taskid=null 88 | hadoop.tasklog.iscleanup=false 89 | hadoop.tasklog.noKeepSplits=4 90 | hadoop.tasklog.totalLogFileSize=100 91 | hadoop.tasklog.purgeLogSplits=true 92 | hadoop.tasklog.logsRetainHours=12 93 | 94 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 95 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 96 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 97 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 98 | 99 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 100 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 101 | 102 | # 103 | # HDFS block state change log from block manager 104 | # 105 | # Uncomment the following to suppress normal block state change 106 | # messages from BlockManager in NameNode. 107 | #log4j.logger.BlockStateChange=WARN 108 | 109 | # 110 | #Security appender 111 | # 112 | hadoop.security.logger=INFO,NullAppender 113 | hadoop.security.log.maxfilesize=256MB 114 | hadoop.security.log.maxbackupindex=20 115 | log4j.category.SecurityLogger=${hadoop.security.logger} 116 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 117 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 118 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 119 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 120 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 121 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 122 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 123 | 124 | # 125 | # Daily Rolling Security appender 126 | # 127 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 128 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 129 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 130 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 131 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 132 | 133 | # 134 | # hadoop configuration logging 135 | # 136 | 137 | # Uncomment the following line to turn off configuration deprecation warnings. 138 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 139 | 140 | # 141 | # hdfs audit logging 142 | # 143 | hdfs.audit.logger=INFO,NullAppender 144 | hdfs.audit.log.maxfilesize=256MB 145 | hdfs.audit.log.maxbackupindex=20 146 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 147 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 148 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 149 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 150 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 151 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 152 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 153 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 154 | 155 | # 156 | # mapred audit logging 157 | # 158 | mapred.audit.logger=INFO,NullAppender 159 | mapred.audit.log.maxfilesize=256MB 160 | mapred.audit.log.maxbackupindex=20 161 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 162 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 163 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 164 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 165 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 166 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 167 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 168 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 169 | 170 | # Custom Logging levels 171 | 172 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 173 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 174 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 175 | 176 | # Jets3t library 177 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 178 | 179 | # 180 | # Event Counter Appender 181 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 182 | # 183 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 184 | 185 | # 186 | # Job Summary Appender 187 | # 188 | # Use following logger to send summary to separate file defined by 189 | # hadoop.mapreduce.jobsummary.log.file : 190 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 191 | # 192 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 193 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 194 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 195 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 196 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 197 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 198 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 199 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 200 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 201 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 202 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 203 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 204 | 205 | # 206 | # Yarn ResourceManager Application Summary Log 207 | # 208 | # Set the ResourceManager summary log filename 209 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 210 | # Set the ResourceManager summary log level and appender 211 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 212 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 213 | 214 | # To enable AppSummaryLogging for the RM, 215 | # set yarn.server.resourcemanager.appsummary.logger to 216 | # ,RMSUMMARY in hadoop-env.sh 217 | 218 | # Appender for ResourceManager Application Summary Log 219 | # Requires the following properties to be set 220 | # - hadoop.log.dir (Hadoop Log directory) 221 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 222 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 223 | 224 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 225 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 226 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 227 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 228 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 229 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 230 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 231 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 232 | 233 | # HS audit log configs 234 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 235 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 236 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 237 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 238 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 239 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 240 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 241 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 242 | 243 | # Http Server Request Logs 244 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 245 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 246 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 247 | #log4j.appender.namenoderequestlog.RetainDays=3 248 | 249 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 250 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 251 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 252 | #log4j.appender.datanoderequestlog.RetainDays=3 253 | 254 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 255 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 256 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 257 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 258 | 259 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 260 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 261 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 262 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 263 | 264 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 265 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 266 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 267 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 268 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/resources/sql.conf: -------------------------------------------------------------------------------- 1 | ############## 2 | # HDFS 3 | ############## 4 | log.hdfs=hdfs://192.168.1.60:9000 5 | 6 | ############## 7 | # SQL 8 | ############## 9 | log.sql1=create /airbook/out1 as select user,count(office) as office from t group by user 10 | log.sql2=create /airbook/out2 as select * from t order by office desc 11 | log.chain=sql2 12 | 13 | 14 | ############## 15 | # VAR 16 | ############## 17 | # log table 18 | log.table.t=#input:#format:#split:#filter 19 | 20 | # split 21 | log.split=| 22 | 23 | # input 24 | log.input=/airbook/20160610.log 25 | 26 | # format 27 | log.format=ip|user|day|office|code 28 | 29 | # log filter 30 | log.filter=^[^#].*?\\|(.*?)\\|(.*?)\\|(\\d+)-.*?\\|\\{"office":"(.*?)","rsCode":(.*?),"pnr":"(.*?)".* 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/sql-egine/src/main/resources/student.txt: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/src/main/resources/teacher.txt: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /src/sql-egine/src/test/java/com/file/FileTableTest.java: -------------------------------------------------------------------------------- 1 | package com.file; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | 7 | public class FileTableTest { 8 | 9 | @Test 10 | public void testFileTable(){ 11 | 12 | String path=FileTableTest.class.getClassLoader().getResource("student.txt").getPath().toString(); 13 | System.out.println(path); 14 | 15 | Table table=new FileTable(path); 16 | assertEquals("表格名称","student",table.getName()); 17 | 18 | System.out.println(table.getName()); 19 | System.out.println(table); 20 | 21 | 22 | String line=table.getRows().get(0); 23 | String column=table.getColumn(line, 1); 24 | System.out.println("获取第一列:"+column); 25 | 26 | column=table.getColumns(line, "*"); 27 | System.out.println("获取所有列:"+column); 28 | 29 | column=table.getColumns(line, "name,grade"); 30 | System.out.println("获取名称和成绩列:"+column); 31 | 32 | 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/sql-egine/src/test/java/com/file/FileTableTest.java.bak: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/src/test/java/com/file/FileTableTest.java.bak -------------------------------------------------------------------------------- /src/sql-egine/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Apache Software Foundation 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Define some default values that can be overridden by system properties 20 | hadoop.root.logger=INFO,console 21 | hadoop.log.dir=. 22 | hadoop.log.file=hadoop.log 23 | 24 | # Define the root logger to the system property "hadoop.root.logger". 25 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 26 | 27 | # Logging Threshold 28 | log4j.threshold=ALL 29 | 30 | # Null Appender 31 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 32 | 33 | # 34 | # Rolling File Appender - cap space usage at 5gb. 35 | # 36 | hadoop.log.maxfilesize=256MB 37 | hadoop.log.maxbackupindex=20 38 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 39 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 40 | 41 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 42 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 43 | 44 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 45 | 46 | # Pattern format: Date LogLevel LoggerName LogMessage 47 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 48 | # Debugging Pattern format 49 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 50 | 51 | 52 | # 53 | # Daily Rolling File Appender 54 | # 55 | 56 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 57 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 58 | 59 | # Rollver at midnight 60 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 61 | 62 | # 30-day backup 63 | #log4j.appender.DRFA.MaxBackupIndex=30 64 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 65 | 66 | # Pattern format: Date LogLevel LoggerName LogMessage 67 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 68 | # Debugging Pattern format 69 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 70 | 71 | 72 | # 73 | # console 74 | # Add "console" to rootlogger above if you want to use this 75 | # 76 | 77 | log4j.appender.console=org.apache.log4j.ConsoleAppender 78 | log4j.appender.console.target=System.err 79 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 80 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 81 | 82 | # 83 | # TaskLog Appender 84 | # 85 | 86 | #Default values 87 | hadoop.tasklog.taskid=null 88 | hadoop.tasklog.iscleanup=false 89 | hadoop.tasklog.noKeepSplits=4 90 | hadoop.tasklog.totalLogFileSize=100 91 | hadoop.tasklog.purgeLogSplits=true 92 | hadoop.tasklog.logsRetainHours=12 93 | 94 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 95 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 96 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 97 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 98 | 99 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 100 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 101 | 102 | # 103 | # HDFS block state change log from block manager 104 | # 105 | # Uncomment the following to suppress normal block state change 106 | # messages from BlockManager in NameNode. 107 | #log4j.logger.BlockStateChange=WARN 108 | 109 | # 110 | #Security appender 111 | # 112 | hadoop.security.logger=INFO,NullAppender 113 | hadoop.security.log.maxfilesize=256MB 114 | hadoop.security.log.maxbackupindex=20 115 | log4j.category.SecurityLogger=${hadoop.security.logger} 116 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 117 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 118 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 119 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 120 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 121 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 122 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 123 | 124 | # 125 | # Daily Rolling Security appender 126 | # 127 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 128 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 129 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 130 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 131 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 132 | 133 | # 134 | # hadoop configuration logging 135 | # 136 | 137 | # Uncomment the following line to turn off configuration deprecation warnings. 138 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 139 | 140 | # 141 | # hdfs audit logging 142 | # 143 | hdfs.audit.logger=INFO,NullAppender 144 | hdfs.audit.log.maxfilesize=256MB 145 | hdfs.audit.log.maxbackupindex=20 146 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 147 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 148 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 149 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 150 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 151 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 152 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 153 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 154 | 155 | # 156 | # mapred audit logging 157 | # 158 | mapred.audit.logger=INFO,NullAppender 159 | mapred.audit.log.maxfilesize=256MB 160 | mapred.audit.log.maxbackupindex=20 161 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 162 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 163 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 164 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 165 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 166 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 167 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 168 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 169 | 170 | # Custom Logging levels 171 | 172 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 173 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 174 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 175 | 176 | # Jets3t library 177 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 178 | 179 | # 180 | # Event Counter Appender 181 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 182 | # 183 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 184 | 185 | # 186 | # Job Summary Appender 187 | # 188 | # Use following logger to send summary to separate file defined by 189 | # hadoop.mapreduce.jobsummary.log.file : 190 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 191 | # 192 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 193 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 194 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 195 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 196 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 197 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 198 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 199 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 200 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 201 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 202 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 203 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 204 | 205 | # 206 | # Yarn ResourceManager Application Summary Log 207 | # 208 | # Set the ResourceManager summary log filename 209 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 210 | # Set the ResourceManager summary log level and appender 211 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 212 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 213 | 214 | # To enable AppSummaryLogging for the RM, 215 | # set yarn.server.resourcemanager.appsummary.logger to 216 | # ,RMSUMMARY in hadoop-env.sh 217 | 218 | # Appender for ResourceManager Application Summary Log 219 | # Requires the following properties to be set 220 | # - hadoop.log.dir (Hadoop Log directory) 221 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 222 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 223 | 224 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 225 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 226 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 227 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 228 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 229 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 230 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 231 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 232 | 233 | # HS audit log configs 234 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 235 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 236 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 237 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 238 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 239 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 240 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 241 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 242 | 243 | # Http Server Request Logs 244 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 245 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 246 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 247 | #log4j.appender.namenoderequestlog.RetainDays=3 248 | 249 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 250 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 251 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 252 | #log4j.appender.datanoderequestlog.RetainDays=3 253 | 254 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 255 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 256 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 257 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 258 | 259 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 260 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 261 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 262 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 263 | 264 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 265 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 266 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 267 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 268 | -------------------------------------------------------------------------------- /src/sql-egine/src/test/resources/student.txt: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/src/test/resources/student.txt.bak: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/src/test/resources/teacher.txt: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /src/sql-egine/src/test/resources/teacher.txt.bak: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /src/sql-egine/target/classes/chain.conf: -------------------------------------------------------------------------------- 1 | ############## 2 | # HDFS 3 | ############## 4 | log.hdfs=hdfs://192.168.1.32:9000 5 | 6 | ############## 7 | # SQL 8 | ############## 9 | 10 | # 简单查询 11 | log.sql1=create /sql/out1 as select * from s order by id desc 12 | 13 | # 简单查询 14 | log.sql2=create /sql/out2 as select id,name,grade from s where id>10 order by grade desc limit 0,10 15 | 16 | # 查询最高的成绩 17 | log.sql3=create /sql/out3 as select max(grade) as grade from s 18 | 19 | # 表连接 20 | log.sql4=create /sql/out4 as select s.id,s.name,s.grade,t.id,t.name from s join t on s.tid=t.id limit 0,10 21 | 22 | # 分组查询 23 | log.sql5=create /sql/out5 as select s.tid,count(s.id) as s.count from s group by s.tid 24 | 25 | # 表连接分组查询 26 | log.sql6=create /sql/out6 as select t.name,count(t.id) as t.count from s join t on s.tid=t.id group by t.id,t.name order by t.count desc limit 0,5 27 | 28 | # log chain 29 | log.chain=sql1,sql2,sql3,sql4,sql5,sql6 30 | 31 | ############## 32 | # VAR 33 | ############## 34 | # log table 35 | log.table.t=/sql/teacher.txt:id|name:#split:#filter 36 | log.table.s=/sql/student.txt:id|name|grade|tid:#split:#filter 37 | 38 | # split 39 | log.split=| 40 | 41 | # log filter 42 | log.filter=(^[^#].*) -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/conf/SqlChain.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/conf/SqlChain.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/conf/SqlConf.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/conf/SqlConf.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/engine/MrEngine.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/engine/MrEngine.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/engine/SqlEngine.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/engine/SqlEngine.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/file/FileTable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/file/FileTable.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/file/HDFSTable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/file/HDFSTable.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/file/Table.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/file/Table.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/hdfs/HDFSHelper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/hdfs/HDFSHelper.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/mr/SortMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/mr/SortMapper.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/mr/SortReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/mr/SortReducer.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/mr/SqlCombiner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/mr/SqlCombiner.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/mr/SqlMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/mr/SqlMapper.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/mr/SqlReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/mr/SqlReducer.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/sql/SqlExeEngine$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/sql/SqlExeEngine$1.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/sql/SqlExeEngine$1On.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/sql/SqlExeEngine$1On.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/sql/SqlExeEngine.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/sql/SqlExeEngine.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/com/sql/SqlParse.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/com/sql/SqlParse.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/log4j.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Apache Software Foundation 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Define some default values that can be overridden by system properties 20 | hadoop.root.logger=INFO,console 21 | hadoop.log.dir=. 22 | hadoop.log.file=hadoop.log 23 | 24 | # Define the root logger to the system property "hadoop.root.logger". 25 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 26 | 27 | # Logging Threshold 28 | log4j.threshold=ALL 29 | 30 | # Null Appender 31 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 32 | 33 | # 34 | # Rolling File Appender - cap space usage at 5gb. 35 | # 36 | hadoop.log.maxfilesize=256MB 37 | hadoop.log.maxbackupindex=20 38 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 39 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 40 | 41 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 42 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 43 | 44 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 45 | 46 | # Pattern format: Date LogLevel LoggerName LogMessage 47 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 48 | # Debugging Pattern format 49 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 50 | 51 | 52 | # 53 | # Daily Rolling File Appender 54 | # 55 | 56 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 57 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 58 | 59 | # Rollver at midnight 60 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 61 | 62 | # 30-day backup 63 | #log4j.appender.DRFA.MaxBackupIndex=30 64 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 65 | 66 | # Pattern format: Date LogLevel LoggerName LogMessage 67 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 68 | # Debugging Pattern format 69 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 70 | 71 | 72 | # 73 | # console 74 | # Add "console" to rootlogger above if you want to use this 75 | # 76 | 77 | log4j.appender.console=org.apache.log4j.ConsoleAppender 78 | log4j.appender.console.target=System.err 79 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 80 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 81 | 82 | # 83 | # TaskLog Appender 84 | # 85 | 86 | #Default values 87 | hadoop.tasklog.taskid=null 88 | hadoop.tasklog.iscleanup=false 89 | hadoop.tasklog.noKeepSplits=4 90 | hadoop.tasklog.totalLogFileSize=100 91 | hadoop.tasklog.purgeLogSplits=true 92 | hadoop.tasklog.logsRetainHours=12 93 | 94 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 95 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 96 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 97 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 98 | 99 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 100 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 101 | 102 | # 103 | # HDFS block state change log from block manager 104 | # 105 | # Uncomment the following to suppress normal block state change 106 | # messages from BlockManager in NameNode. 107 | #log4j.logger.BlockStateChange=WARN 108 | 109 | # 110 | #Security appender 111 | # 112 | hadoop.security.logger=INFO,NullAppender 113 | hadoop.security.log.maxfilesize=256MB 114 | hadoop.security.log.maxbackupindex=20 115 | log4j.category.SecurityLogger=${hadoop.security.logger} 116 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 117 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 118 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 119 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 120 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 121 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 122 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 123 | 124 | # 125 | # Daily Rolling Security appender 126 | # 127 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 128 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 129 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 130 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 131 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 132 | 133 | # 134 | # hadoop configuration logging 135 | # 136 | 137 | # Uncomment the following line to turn off configuration deprecation warnings. 138 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 139 | 140 | # 141 | # hdfs audit logging 142 | # 143 | hdfs.audit.logger=INFO,NullAppender 144 | hdfs.audit.log.maxfilesize=256MB 145 | hdfs.audit.log.maxbackupindex=20 146 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 147 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 148 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 149 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 150 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 151 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 152 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 153 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 154 | 155 | # 156 | # mapred audit logging 157 | # 158 | mapred.audit.logger=INFO,NullAppender 159 | mapred.audit.log.maxfilesize=256MB 160 | mapred.audit.log.maxbackupindex=20 161 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 162 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 163 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 164 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 165 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 166 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 167 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 168 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 169 | 170 | # Custom Logging levels 171 | 172 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 173 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 174 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 175 | 176 | # Jets3t library 177 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 178 | 179 | # 180 | # Event Counter Appender 181 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 182 | # 183 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 184 | 185 | # 186 | # Job Summary Appender 187 | # 188 | # Use following logger to send summary to separate file defined by 189 | # hadoop.mapreduce.jobsummary.log.file : 190 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 191 | # 192 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 193 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 194 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 195 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 196 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 197 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 198 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 199 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 200 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 201 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 202 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 203 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 204 | 205 | # 206 | # Yarn ResourceManager Application Summary Log 207 | # 208 | # Set the ResourceManager summary log filename 209 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 210 | # Set the ResourceManager summary log level and appender 211 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 212 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 213 | 214 | # To enable AppSummaryLogging for the RM, 215 | # set yarn.server.resourcemanager.appsummary.logger to 216 | # ,RMSUMMARY in hadoop-env.sh 217 | 218 | # Appender for ResourceManager Application Summary Log 219 | # Requires the following properties to be set 220 | # - hadoop.log.dir (Hadoop Log directory) 221 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 222 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 223 | 224 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 225 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 226 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 227 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 228 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 229 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 230 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 231 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 232 | 233 | # HS audit log configs 234 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 235 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 236 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 237 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 238 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 239 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 240 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 241 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 242 | 243 | # Http Server Request Logs 244 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 245 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 246 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 247 | #log4j.appender.namenoderequestlog.RetainDays=3 248 | 249 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 250 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 251 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 252 | #log4j.appender.datanoderequestlog.RetainDays=3 253 | 254 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 255 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 256 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 257 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 258 | 259 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 260 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 261 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 262 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 263 | 264 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 265 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 266 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 267 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 268 | -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$CachedUid.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$CachedUid.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$CacheManipulator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$CacheManipulator.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$CachedName.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$CachedName.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$IdCache.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$IdCache.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$NoMlockCacheManipulator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$NoMlockCacheManipulator.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$POSIX.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$Windows$AccessRight.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$Windows$AccessRight.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$Windows.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO$Windows.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/classes/org/apache/hadoop/io/nativeio/NativeIO.class -------------------------------------------------------------------------------- /src/sql-egine/target/classes/sql.conf: -------------------------------------------------------------------------------- 1 | ############## 2 | # HDFS 3 | ############## 4 | log.hdfs=hdfs://192.168.1.60:9000 5 | 6 | ############## 7 | # SQL 8 | ############## 9 | log.sql1=create /airbook/out1 as select user,count(office) as office from t group by user 10 | log.sql2=create /airbook/out2 as select * from t order by office desc 11 | log.chain=sql2 12 | 13 | 14 | ############## 15 | # VAR 16 | ############## 17 | # log table 18 | log.table.t=#input:#format:#split:#filter 19 | 20 | # split 21 | log.split=| 22 | 23 | # input 24 | log.input=/airbook/20160610.log 25 | 26 | # format 27 | log.format=ip|user|day|office|code 28 | 29 | # log filter 30 | log.filter=^[^#].*?\\|(.*?)\\|(.*?)\\|(\\d+)-.*?\\|\\{"office":"(.*?)","rsCode":(.*?),"pnr":"(.*?)".* 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/sql-egine/target/classes/student.txt: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/target/classes/teacher.txt: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/com/file/FileTableTest.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/test-classes/com/file/FileTableTest.class -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/com/file/FileTableTest.java.bak: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mircode/sql-engine/344939f926467a1df31b4852995ce87d4df23280/src/sql-egine/target/test-classes/com/file/FileTableTest.java.bak -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/log4j.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Apache Software Foundation 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Define some default values that can be overridden by system properties 20 | hadoop.root.logger=INFO,console 21 | hadoop.log.dir=. 22 | hadoop.log.file=hadoop.log 23 | 24 | # Define the root logger to the system property "hadoop.root.logger". 25 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 26 | 27 | # Logging Threshold 28 | log4j.threshold=ALL 29 | 30 | # Null Appender 31 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 32 | 33 | # 34 | # Rolling File Appender - cap space usage at 5gb. 35 | # 36 | hadoop.log.maxfilesize=256MB 37 | hadoop.log.maxbackupindex=20 38 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 39 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 40 | 41 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 42 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 43 | 44 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 45 | 46 | # Pattern format: Date LogLevel LoggerName LogMessage 47 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 48 | # Debugging Pattern format 49 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 50 | 51 | 52 | # 53 | # Daily Rolling File Appender 54 | # 55 | 56 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 57 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 58 | 59 | # Rollver at midnight 60 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 61 | 62 | # 30-day backup 63 | #log4j.appender.DRFA.MaxBackupIndex=30 64 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 65 | 66 | # Pattern format: Date LogLevel LoggerName LogMessage 67 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 68 | # Debugging Pattern format 69 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 70 | 71 | 72 | # 73 | # console 74 | # Add "console" to rootlogger above if you want to use this 75 | # 76 | 77 | log4j.appender.console=org.apache.log4j.ConsoleAppender 78 | log4j.appender.console.target=System.err 79 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 80 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 81 | 82 | # 83 | # TaskLog Appender 84 | # 85 | 86 | #Default values 87 | hadoop.tasklog.taskid=null 88 | hadoop.tasklog.iscleanup=false 89 | hadoop.tasklog.noKeepSplits=4 90 | hadoop.tasklog.totalLogFileSize=100 91 | hadoop.tasklog.purgeLogSplits=true 92 | hadoop.tasklog.logsRetainHours=12 93 | 94 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 95 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 96 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 97 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 98 | 99 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 100 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 101 | 102 | # 103 | # HDFS block state change log from block manager 104 | # 105 | # Uncomment the following to suppress normal block state change 106 | # messages from BlockManager in NameNode. 107 | #log4j.logger.BlockStateChange=WARN 108 | 109 | # 110 | #Security appender 111 | # 112 | hadoop.security.logger=INFO,NullAppender 113 | hadoop.security.log.maxfilesize=256MB 114 | hadoop.security.log.maxbackupindex=20 115 | log4j.category.SecurityLogger=${hadoop.security.logger} 116 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 117 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 118 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 119 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 120 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 121 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 122 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 123 | 124 | # 125 | # Daily Rolling Security appender 126 | # 127 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 128 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 129 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 130 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 131 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 132 | 133 | # 134 | # hadoop configuration logging 135 | # 136 | 137 | # Uncomment the following line to turn off configuration deprecation warnings. 138 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 139 | 140 | # 141 | # hdfs audit logging 142 | # 143 | hdfs.audit.logger=INFO,NullAppender 144 | hdfs.audit.log.maxfilesize=256MB 145 | hdfs.audit.log.maxbackupindex=20 146 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 147 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 148 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 149 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 150 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 151 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 152 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 153 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 154 | 155 | # 156 | # mapred audit logging 157 | # 158 | mapred.audit.logger=INFO,NullAppender 159 | mapred.audit.log.maxfilesize=256MB 160 | mapred.audit.log.maxbackupindex=20 161 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 162 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 163 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 164 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 165 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 166 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 167 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 168 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 169 | 170 | # Custom Logging levels 171 | 172 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 173 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 174 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 175 | 176 | # Jets3t library 177 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 178 | 179 | # 180 | # Event Counter Appender 181 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 182 | # 183 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 184 | 185 | # 186 | # Job Summary Appender 187 | # 188 | # Use following logger to send summary to separate file defined by 189 | # hadoop.mapreduce.jobsummary.log.file : 190 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 191 | # 192 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 193 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 194 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 195 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 196 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 197 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 198 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 199 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 200 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 201 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 202 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 203 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 204 | 205 | # 206 | # Yarn ResourceManager Application Summary Log 207 | # 208 | # Set the ResourceManager summary log filename 209 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 210 | # Set the ResourceManager summary log level and appender 211 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 212 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 213 | 214 | # To enable AppSummaryLogging for the RM, 215 | # set yarn.server.resourcemanager.appsummary.logger to 216 | # ,RMSUMMARY in hadoop-env.sh 217 | 218 | # Appender for ResourceManager Application Summary Log 219 | # Requires the following properties to be set 220 | # - hadoop.log.dir (Hadoop Log directory) 221 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 222 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 223 | 224 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 225 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 226 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 227 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 228 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 229 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 230 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 231 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 232 | 233 | # HS audit log configs 234 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 235 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 236 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 237 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 238 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 239 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 240 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 241 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 242 | 243 | # Http Server Request Logs 244 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 245 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 246 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 247 | #log4j.appender.namenoderequestlog.RetainDays=3 248 | 249 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 250 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 251 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 252 | #log4j.appender.datanoderequestlog.RetainDays=3 253 | 254 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 255 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 256 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 257 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 258 | 259 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 260 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 261 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 262 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 263 | 264 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 265 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 266 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 267 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 268 | -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/student.txt: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/student.txt.bak: -------------------------------------------------------------------------------- 1 | #id|name|grade|tid 2 | 0|student0|90|2 3 | 1|student1|54|9 4 | 2|student2|34|2 5 | 3|student3|21|6 6 | 4|student4|12|0 7 | 5|student5|43|3 8 | 6|student6|87|3 9 | 7|student7|3|1 10 | 8|student8|58|8 11 | 9|student9|41|9 12 | 10|student10|47|7 13 | 11|student11|35|8 14 | 12|student12|42|7 15 | 13|student13|17|4 16 | 14|student14|72|4 17 | 15|student15|15|0 18 | 16|student16|5|6 19 | 17|student17|84|6 20 | 18|student18|78|1 21 | 19|student19|49|0 22 | 20|student20|83|1 23 | 21|student21|67|7 24 | 22|student22|94|0 25 | 23|student23|47|4 26 | 24|student24|11|8 27 | 25|student25|2|4 28 | 26|student26|99|3 29 | 27|student27|43|9 30 | 28|student28|96|7 31 | 29|student29|89|3 32 | 30|student30|85|9 33 | 31|student31|84|9 34 | 32|student32|18|4 35 | 33|student33|0|4 36 | 34|student34|46|3 37 | 35|student35|45|5 38 | 36|student36|34|5 39 | 37|student37|91|4 40 | 38|student38|4|8 41 | 39|student39|94|8 42 | 40|student40|13|8 43 | 41|student41|6|9 44 | 42|student42|93|7 45 | 43|student43|15|4 46 | 44|student44|61|5 47 | 45|student45|25|1 48 | 46|student46|25|8 49 | 47|student47|8|5 50 | 48|student48|32|0 51 | 49|student49|69|0 52 | 50|student50|48|9 53 | 51|student51|53|5 54 | 52|student52|92|4 55 | 53|student53|78|9 56 | 54|student54|83|6 57 | 55|student55|33|8 58 | 56|student56|32|5 59 | 57|student57|30|7 60 | 58|student58|60|7 61 | 59|student59|43|0 62 | 60|student60|50|6 63 | 61|student61|55|6 64 | 62|student62|57|6 65 | 63|student63|84|0 66 | 64|student64|15|9 67 | 65|student65|64|0 68 | 66|student66|77|9 69 | 67|student67|12|3 70 | 68|student68|47|1 71 | 69|student69|44|4 72 | 70|student70|6|5 73 | 71|student71|21|3 74 | 72|student72|7|7 75 | 73|student73|1|0 76 | 74|student74|8|0 77 | 75|student75|77|0 78 | 76|student76|64|2 79 | 77|student77|24|8 80 | 78|student78|77|8 81 | 79|student79|44|6 82 | 80|student80|2|8 83 | 81|student81|31|6 84 | 82|student82|32|6 85 | 83|student83|81|7 86 | 84|student84|0|7 87 | 85|student85|39|4 88 | 86|student86|27|3 89 | 87|student87|64|0 90 | 88|student88|76|3 91 | 89|student89|41|6 92 | 90|student90|32|5 93 | 91|student91|32|4 94 | 92|student92|56|8 95 | 93|student93|69|4 96 | 94|student94|61|3 97 | 95|student95|86|1 98 | 96|student96|82|3 99 | 97|student97|73|5 100 | 98|student98|98|2 101 | 99|student99|7|1 -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/teacher.txt: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 -------------------------------------------------------------------------------- /src/sql-egine/target/test-classes/teacher.txt.bak: -------------------------------------------------------------------------------- 1 | #id|name 2 | 0|teacher0 3 | 1|teacher1 4 | 2|teacher2 5 | 3|teacher3 6 | 4|teacher4 7 | 5|teacher5 8 | 6|teacher6 9 | 7|teacher7 10 | 8|teacher8 11 | 9|teacher9 --------------------------------------------------------------------------------