├── .gitignore ├── CMakeLists.txt ├── Makefile ├── MushroomCloud ├── Manifest.txt ├── MushroomCloud.iml ├── README.md ├── lib │ ├── expression.jar │ ├── java-json.jar │ └── jsqlparser.jar └── src │ └── edu │ └── buffalo │ └── cse562 │ ├── LeafValueComparator.java │ ├── Main.java │ ├── ParseTreeEvaluator.java │ ├── ParseTreeGenerator.java │ ├── ParseTreeOptimizer.java │ ├── exceptions │ └── UnsupportedStatementException.java │ ├── operators │ ├── CrossProductOperator.java │ ├── DistinctOperator.java │ ├── ExternalSortOperator.java │ ├── GraceHashJoinOperator.java │ ├── LimitOperator.java │ ├── Operator.java │ ├── OrderByOperator.java │ ├── ProjectScanOperator.java │ ├── ProjectionGroupByAggregateOperator.java │ ├── ProjectionOperator.java │ ├── ScanOperator.java │ ├── SelectionOperator.java │ ├── SortMergeJoinOperator.java │ └── UnionOperator.java │ └── schema │ ├── ColumnInfo.java │ ├── ColumnWithType.java │ └── Schema.java ├── README.md ├── Shell.py ├── Valkyrie ├── CMakeLists.txt ├── Makefile ├── include │ ├── Codegen.h │ ├── DataTypes.h │ ├── Expression.h │ ├── ExpressionParser.h │ ├── JoinOperator.h │ ├── Operator.h │ ├── Parser.h │ ├── PrintOperator.h │ ├── ProjectionOperator.h │ ├── ScanOperator.h │ ├── Schema.h │ ├── SelectOperator.h │ └── rapidjson │ │ ├── allocators.h │ │ ├── document.h │ │ ├── encodedstream.h │ │ ├── encodings.h │ │ ├── error │ │ ├── en.h │ │ └── error.h │ │ ├── filereadstream.h │ │ ├── filewritestream.h │ │ ├── internal │ │ ├── biginteger.h │ │ ├── diyfp.h │ │ ├── dtoa.h │ │ ├── ieee754.h │ │ ├── itoa.h │ │ ├── meta.h │ │ ├── pow10.h │ │ ├── stack.h │ │ ├── strfunc.h │ │ ├── strtod.h │ │ └── swap.h │ │ ├── memorybuffer.h │ │ ├── memorystream.h │ │ ├── msinttypes │ │ ├── inttypes.h │ │ └── stdint.h │ │ ├── pointer.h │ │ ├── prettywriter.h │ │ ├── rapidjson.h │ │ ├── reader.h │ │ ├── stringbuffer.h │ │ └── writer.h └── src │ ├── Codegen.cpp │ ├── Expression.cpp │ ├── ExpressionParser.cpp │ ├── JoinOperator.cpp │ ├── Main.cpp │ ├── Operator.cpp │ ├── Parser.cpp │ ├── PrintOperator.cpp │ ├── ProjectionOperator.cpp │ ├── ScanOperator.cpp │ ├── Schema.cpp │ └── SelectOperator.cpp ├── ValkyrieDB.sh ├── allexps.sh ├── exp.sh └── test ├── cprogs ├── q1 ├── q1.c ├── q2.c ├── q3.c └── q4.cpp ├── sql ├── ddl-schema.sql ├── test.sql ├── testcases │ ├── 1.sql │ ├── 2.sql │ ├── 3.sql │ ├── 4.sql │ └── 5.sql ├── tpch1.sql ├── tpch10.sql ├── tpch12.sql ├── tpch3.sql ├── tpch5.sql ├── tpch6.sql └── tpch_schemas.sql └── tpch-data-setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | MushroomCloud/bin/ 3 | Valkyrie/build/ 4 | 5 | .idea/ 6 | 7 | perf.data 8 | perf.data.old 9 | 10 | *.tbl 11 | *.out 12 | 13 | test/tpch.db 14 | test/tpch-dbgen -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(ValkyrieDB) 3 | 4 | add_subdirectory(Valkyrie) -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | INC_DIR = ./Valkyrie/include 2 | SRC_DIR = ./Valkyrie/src 3 | OBJ_DIR = ./Valkyrie/object 4 | 5 | SRC_FILES = $(wildcard $(SRC_DIR)/*.cpp) 6 | OBJ_FILES = $(SRC_FILES:.cpp=.o) 7 | OBJ_PATH = $(patsubst $(SRC_DIR)/%,$(OBJ_DIR)/%,$(OBJ_FILES)) 8 | 9 | LIBS= 10 | CC=clang++-3.5 11 | CFLAGS=-rdynamic -g -O3 -m64 `llvm-config-3.5 --cxxflags --ldflags --system-libs --libs core mcjit native bitwriter` 12 | 13 | all: init mushroomcloud valkyrie 14 | 15 | init: 16 | mkdir -p ./bin 17 | mkdir -p MushroomCloud/bin 18 | 19 | mushroomcloud: 20 | javac -d MushroomCloud/bin/ -cp MushroomCloud/src:"MushroomCloud/lib/*" -sourcepath MushroomCloud/src MushroomCloud/src/edu/buffalo/cse562/Main.java 21 | jar cvfm bin/MushroomCloud.jar MushroomCloud/Manifest.txt -C MushroomCloud/bin/ . 22 | cp -a MushroomCloud/lib/. ./bin/ 23 | 24 | valkyrie: 25 | $(CC) $(SRC_DIR)/*.cpp $(CFLAGS) -o bin/llvmruntime -I$(INC_DIR) 26 | 27 | clean: 28 | rm -r MushroomCloud/bin 29 | rm -r ./bin 30 | -------------------------------------------------------------------------------- /MushroomCloud/Manifest.txt: -------------------------------------------------------------------------------- 1 | Main-Class: edu/buffalo/cse562/Main 2 | Class-Path: ./jsqlparser.jar ./expression.jar ./java-json.jar 3 | -------------------------------------------------------------------------------- /MushroomCloud/MushroomCloud.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /MushroomCloud/README.md: -------------------------------------------------------------------------------- 1 | # SQL Evaluator 2 | A SQL evaluator done as part of a graduate database class 3 | 4 | This project translates SQL to an abstract syntax tree (AST) representation and then applies relational algebra rewrites to find an optimum query evaluation strategy. Efficient joins and external sort algorithms enable it to work with huge datasets even on limited memory. 5 | 6 | # Build and run 7 | 8 | To build, just run the build script - 9 | 10 | ``` 11 | > ./build.sh 12 | ``` 13 | 14 | Do not forget to add executable permission to the script using ```chmod +x``` 15 | 16 | This will compile all classes in the ```src``` directory and place them in an automatically created ```bin```directory. 17 | 18 | To run, you can use the included run script - 19 | 20 | ``` 21 | > ./run.sh 22 | ``` 23 | 24 | # Sample data and queries 25 | 26 | Several sample datasets and sql queries are included. These can be found in the ```data``` folder and ```sql`` folders respectively. 27 | 28 | There is a set of small sanity check relations ```r, s, t``` comprising of just integer data. A bite-sized TPC-H dataset has been included to test TPC-H queries against. To generate a larger dataset please use a DBGen program for the TPC-H benchmark. 29 | 30 | This is one that has been tested and works. 31 | https://github.com/electrum/tpch-dbgen 32 | 33 | # Syntax 34 | 35 | ``` 36 | > run.sh --data [ --swap ] ... 37 | ``` 38 | 39 | The data flag points to the data directory. The swap flag points to a directory used for out of memory operations like external sorts. 40 | 41 | # Example 42 | 43 | ``` 44 | > ./run.sh --data data sql/tpch1.sql 45 | A|F|3608|3617399.8|3415815.6154|3550622.3881|25.5887|25655.3177|0.0535|141 46 | N|F|98|96050.28|93793.9484|94868.9501|32.6667|32016.76|0.0233|3 47 | N|O|7917|7922719.62|7540013.3753|7850451.2831|25.9574|25976.1299|0.0487|305 48 | R|F|3269|3260914.61|3079298.8793|3200628.9333|24.3955|24335.1837|0.0516|134 49 | ``` 50 | 51 | # Additional Flags 52 | 53 | SQL-Parser supports a few additional flags to generate output and show debugging information. 54 | 55 | ```--degug``` flag enables debugging information like generated query plans, optimized query plans and execution times. 56 | 57 | ```-q``` quiet mode, results are not output to the console. 58 | 59 | ```--fileout ``` results are written to a file named ```.out``` in the specified directory, which will be created if it doesn't exist. 60 | 61 | ```--time ``` Query plan generation time and query execution times are written as a ```|``` separated tuple to a file named ```.stat``` in the specified directory, which will be created if it doesn't exist. 62 | -------------------------------------------------------------------------------- /MushroomCloud/lib/expression.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Legacy25/ValkyrieDB/c2adee18658658e78b7a4b464c8db91657fe58c1/MushroomCloud/lib/expression.jar -------------------------------------------------------------------------------- /MushroomCloud/lib/java-json.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Legacy25/ValkyrieDB/c2adee18658658e78b7a4b464c8db91657fe58c1/MushroomCloud/lib/java-json.jar -------------------------------------------------------------------------------- /MushroomCloud/lib/jsqlparser.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Legacy25/ValkyrieDB/c2adee18658658e78b7a4b464c8db91657fe58c1/MushroomCloud/lib/jsqlparser.jar -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/LeafValueComparator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Comparator; 5 | import java.util.HashMap; 6 | 7 | import edu.buffalo.cse562.schema.Schema; 8 | import net.sf.jsqlparser.expression.DateValue; 9 | import net.sf.jsqlparser.expression.DoubleValue; 10 | import net.sf.jsqlparser.expression.LeafValue; 11 | import net.sf.jsqlparser.expression.LongValue; 12 | import net.sf.jsqlparser.expression.StringValue; 13 | import net.sf.jsqlparser.expression.LeafValue.InvalidLeaf; 14 | import net.sf.jsqlparser.statement.select.OrderByElement; 15 | 16 | public class LeafValueComparator implements Comparator { 17 | 18 | 19 | private Schema schema; 20 | 21 | /* Holds the sort key attributes */ 22 | private ArrayList arguments; 23 | 24 | /* Holds the indexes of the sort key attributes */ 25 | private HashMap columnIndexes; 26 | 27 | 28 | public LeafValueComparator(ArrayList arguments , 29 | Schema schema) { 30 | 31 | this.arguments = arguments; 32 | this.schema = schema; 33 | 34 | columnIndexes = new HashMap(); 35 | 36 | findColumns(); 37 | 38 | } 39 | 40 | 41 | /* 42 | * Helper function to find the appropriate column indexes on which to sort 43 | */ 44 | private void findColumns() { 45 | 46 | for(OrderByElement o:arguments) { 47 | String columnName = o.getExpression().toString(); 48 | 49 | for(int i=0; i= 0 51 | || columnName.toLowerCase().indexOf(schema.getColumns().get(i).getColumnName().toLowerCase()) >= 0) { 52 | 53 | columnIndexes.put(o, i); 54 | break; 55 | 56 | } 57 | } 58 | } 59 | 60 | } 61 | 62 | 63 | 64 | @Override 65 | public int compare(LeafValue[] o1, LeafValue[] o2) { 66 | String type = ""; 67 | int returnVal = 0; 68 | 69 | for(int i=arguments.size()-1; i>=0; i--) { 70 | OrderByElement o = arguments.get(i); 71 | int column = columnIndexes.get(o); 72 | boolean isAsc = o.isAsc(); 73 | LeafValue element = o1[column]; 74 | 75 | if(element instanceof LongValue) { 76 | type = "int"; 77 | } 78 | else if(element instanceof DoubleValue) { 79 | type = "decimal"; 80 | } 81 | else if(element instanceof StringValue) { 82 | type = "string"; 83 | } 84 | else if(element instanceof DateValue) { 85 | type = "date"; 86 | } 87 | 88 | switch(type) { 89 | case "int": 90 | case "decimal": 91 | try { 92 | if(o1[column].toDouble() == o2[column].toDouble()) 93 | continue; 94 | if(o1[column].toDouble() > o2[column].toDouble()) { 95 | if(isAsc) { 96 | returnVal = 1; 97 | } 98 | else { 99 | returnVal = -1; 100 | } 101 | } 102 | else { 103 | if(isAsc) { 104 | returnVal = -1; 105 | } 106 | else { 107 | returnVal = 1; 108 | } 109 | } 110 | break; 111 | } catch (InvalidLeaf e) { 112 | break; 113 | } 114 | case "string": 115 | case "date": 116 | if(o1[column].toString().equalsIgnoreCase(o2[column].toString())) { 117 | continue; 118 | } 119 | if(o1[column].toString().compareToIgnoreCase(o2[column].toString()) > 0) { 120 | if(isAsc) { 121 | returnVal = 1; 122 | } 123 | else { 124 | returnVal = -1; 125 | } 126 | } 127 | else { 128 | if(isAsc) { 129 | returnVal = -1; 130 | } 131 | else { 132 | returnVal = 1; 133 | } 134 | } 135 | break; 136 | default: 137 | /* Handle */ 138 | } 139 | } 140 | 141 | return returnVal; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/Main.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SQL Evaluator Engine 3 | * Authors: Arindam Nandi 4 | * Saptarshi Bhattacharjee 5 | * Sayaritra Pal 6 | * 7 | * Spring 2015 8 | */ 9 | 10 | package edu.buffalo.cse562; 11 | 12 | import java.io.BufferedWriter; 13 | import java.io.File; 14 | import java.io.FileWriter; 15 | import java.io.IOException; 16 | import java.nio.file.Files; 17 | import java.util.ArrayList; 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | //import java.util.function.Consumer; 21 | 22 | import org.json.JSONObject; 23 | 24 | import edu.buffalo.cse562.operators.Operator; 25 | 26 | public class Main { 27 | 28 | public static final int BILLION = 1000*1000*1000; 29 | 30 | /* 31 | * Stores the swap directory 32 | * Need application-wide access to this, so a static global 33 | */ 34 | public static File SWAP = null; 35 | 36 | /* 37 | * Initialize a file name counter required for bookkeeping 38 | * in external k-way merge sort 39 | */ 40 | public static int FILE_UUID = 0; 41 | 42 | /* 43 | * Controls the size of the I/O buffer used to retrieve 44 | * records from file to memory. Varying this gives some 45 | * interesting variations in query processing time 46 | */ 47 | public static int BLOCK = 10000; 48 | 49 | /* 50 | * Provides application wide access to information 51 | * regarding whether there is a memory limit, 52 | * this is of use to the parse-tree optimizer 53 | * in choosing non-blocking operators 54 | */ 55 | public static boolean MEMORY_LIMITED = false; 56 | 57 | /* 58 | * Enabled by setting the --debug flag on the CLI. 59 | * Mainly used to print out various debugging info 60 | * like query plans and processing times 61 | */ 62 | public static boolean DEBUG = false; 63 | 64 | /* 65 | * --fileout flag redirects the results to files 66 | * instead of the console 67 | */ 68 | public static boolean FILE_OUTPUT = false; 69 | public static File FILE_OUTPUT_DIR = null; 70 | 71 | /* 72 | * --time flag prints out query execution times to a file 73 | */ 74 | public static boolean TIME_OUTPUT = false; 75 | public static File TIME_OUTPUT_DIR = null; 76 | 77 | 78 | /* 79 | * -q Quiet mode 80 | */ 81 | public static boolean QUIET = false; 82 | 83 | /* 84 | * Execute mode, either dumps a representation of the query plan 85 | * or executes the query 86 | */ 87 | public static boolean EXECUTE = false; 88 | 89 | public static boolean PUSH_DOWN_PROJECTIONS = false; 90 | 91 | public static void main(String[] args) { 92 | 93 | DEBUG = false; 94 | 95 | /* Stores the data directories */ 96 | ArrayList dataDirs = new ArrayList(); 97 | 98 | /* Stores the SQL files */ 99 | ArrayList sqlFiles = new ArrayList(); 100 | 101 | /* Stores query generation times for each query */ 102 | ArrayList qgenTime = new ArrayList(); 103 | 104 | /* Stores query execution times for each query */ 105 | ArrayList qexecTime = new ArrayList(); 106 | 107 | /* 108 | * CLI argument parsing 109 | */ 110 | for(int i=0; i parseTreeList = new ArrayList(); 168 | for(int i = 0; i < sqlFiles.size(); i++) { 169 | File f = sqlFiles.get(i); 170 | long localStart = System.nanoTime(); 171 | parseTreeList.add(ParseTreeGenerator.generate(dataDirs, f)); 172 | 173 | /* Compute the generation time for this query */ 174 | qgenTime.add((double) (System.nanoTime() - localStart)/BILLION); 175 | } 176 | 177 | /* Optimize each AST */ 178 | for(int i = 0; i < parseTreeList.size(); i++) { 179 | Operator parseTree = parseTreeList.get(i); 180 | long localStart = System.nanoTime(); 181 | parseTreeList.set(i, ParseTreeOptimizer.optimize(parseTree)); 182 | 183 | /* Compute the optimization time for this query 184 | * and add it to the generation time 185 | */ 186 | qgenTime.set(i, qgenTime.get(i) + (double) (System.nanoTime() - localStart)/BILLION); 187 | 188 | /* 189 | * Display the query plan 190 | */ 191 | /*if(Main.DEBUG) { 192 | parseTreeList.forEach(new Consumer() { 193 | @Override 194 | public void accept(Operator t) { 195 | System.out.println(t.getSchema()); 196 | } 197 | }); 198 | }*/ 199 | } 200 | 201 | /* 202 | * Optimized query plans ready 203 | */ 204 | 205 | long totalGenerateTime = System.nanoTime(); 206 | 207 | if(Main.EXECUTE){ 208 | /* Now evaluate each parse-tree */ 209 | for(int i=0; i< parseTreeList.size(); i++) { 210 | long localStart = System.nanoTime(); 211 | 212 | if(parseTreeList.get(i) != null) { 213 | if(FILE_OUTPUT) { 214 | ParseTreeEvaluator.output( 215 | parseTreeList.get(i), 216 | new File( 217 | FILE_OUTPUT_DIR, 218 | sqlFiles.get(i).getName().split(".sql")[0] + ".out" 219 | ) 220 | ); 221 | } else { 222 | ParseTreeEvaluator.output(parseTreeList.get(i)); 223 | } 224 | } 225 | 226 | qexecTime.add((double) (System.nanoTime() - localStart)/BILLION); 227 | } 228 | } else { 229 | for (Operator operator : parseTreeList) { 230 | if(operator != null) { 231 | Map root = new HashMap(); 232 | root.put("TYPE", "PRINT"); 233 | root.put("SRC", operator.getDetails()); 234 | root.put("EXPRESSION", new ArrayList()); 235 | System.out.println(new JSONObject(root)); 236 | } 237 | } 238 | } 239 | 240 | 241 | double totalGenTime = (double) (totalGenerateTime - globalStart)/BILLION; 242 | double totalExecTime = (double) (System.nanoTime() - totalGenerateTime)/BILLION; 243 | 244 | /* DEBUG */ 245 | /* Show query times */ 246 | if(Main.DEBUG) { 247 | 248 | System.err.println("TOTAL GENERATE TIME: " 249 | + (totalGenTime) 250 | + "s"); 251 | 252 | System.err.println("TOTAL EXECUTION TIME: " 253 | + (totalExecTime) 254 | + "s"); 255 | 256 | } 257 | 258 | if(Main.TIME_OUTPUT) { 259 | for(int i = 0; i < sqlFiles.size(); i++) { 260 | try { 261 | if(!TIME_OUTPUT_DIR.exists()) { 262 | TIME_OUTPUT_DIR.mkdirs(); 263 | } 264 | BufferedWriter bw = new BufferedWriter( 265 | new FileWriter( 266 | new File( 267 | TIME_OUTPUT_DIR, 268 | sqlFiles.get(i).getName().split(".sql")[0]+".stat") 269 | ) 270 | ); 271 | 272 | bw.write(qgenTime.get(i)+"|"+qexecTime.get(i)); 273 | bw.flush(); 274 | bw.close(); 275 | } catch (IOException e) { 276 | e.printStackTrace(); 277 | } 278 | } 279 | } 280 | } 281 | } 282 | 283 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/ParseTreeEvaluator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562; 2 | 3 | import java.io.BufferedWriter; 4 | import java.io.File; 5 | import java.io.FileWriter; 6 | import java.io.IOException; 7 | import java.text.DecimalFormat; 8 | 9 | import net.sf.jsqlparser.expression.DoubleValue; 10 | import net.sf.jsqlparser.expression.LeafValue; 11 | import net.sf.jsqlparser.expression.LeafValue.InvalidLeaf; 12 | import net.sf.jsqlparser.expression.StringValue; 13 | import edu.buffalo.cse562.operators.Operator; 14 | 15 | public class ParseTreeEvaluator { 16 | 17 | public static void output(Operator operator) { 18 | output(operator, null); 19 | } 20 | 21 | public static void output(Operator operator, File f) { 22 | 23 | if(operator == null) { 24 | return; 25 | } 26 | 27 | operator.initialize(); 28 | 29 | BufferedWriter bw = null; 30 | if(f != null) { 31 | try { 32 | if(!f.getParentFile().exists()) { 33 | if(!f.getParentFile().mkdirs()) { 34 | System.err.println("Couldn't create output directory! Exiting..."); 35 | System.exit(1); 36 | } 37 | } 38 | bw = new BufferedWriter(new FileWriter(f)); 39 | } catch (IOException e) { 40 | e.printStackTrace(); 41 | } 42 | } 43 | 44 | LeafValue res[] = null; 45 | /* Keep getting a tuple and displaying it till we exhaust the root operator */ 46 | while((res = operator.readOneTuple()) != null) { 47 | String s = tupleToString(res); 48 | if(bw == null) { 49 | if(!Main.QUIET) System.out.println(s); 50 | } 51 | else { 52 | try { 53 | bw.write(s+"\n"); 54 | } catch (IOException e) { 55 | e.printStackTrace(); 56 | } 57 | } 58 | } 59 | 60 | if(bw != null) { 61 | try { 62 | bw.flush(); 63 | bw.close(); 64 | } catch (IOException e) { 65 | e.printStackTrace(); 66 | } 67 | } 68 | } 69 | 70 | 71 | /* Formatting logic for tuples to strings */ 72 | public static String tupleToString(LeafValue res[]) { 73 | boolean flag = false; 74 | String result = ""; 75 | DecimalFormat twoDForm = new DecimalFormat("#.############"); 76 | 77 | for(int i=0; i getDetails() { 195 | Map map = new HashMap(); 196 | List expList = new ArrayList(); 197 | List srcs = new ArrayList(); 198 | srcs.add(child1.getDetails()); 199 | srcs.add(child2.getDetails()); 200 | map.put("TYPE", "PRODUCT"); 201 | map.put("EXPRESSION", expList); 202 | map.put("SRC", srcs); 203 | return map; 204 | } 205 | 206 | } 207 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/DistinctOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.HashSet; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import net.sf.jsqlparser.expression.LeafValue; 11 | import edu.buffalo.cse562.schema.Schema; 12 | 13 | public class DistinctOperator implements Operator { 14 | 15 | /* 16 | * Distinct Operator 17 | * Scans over the child operator and 18 | * emits only one copy of same tuples 19 | * 20 | * Constructor Variables 21 | * The child operator 22 | * 23 | * Working Set Size - The size of the seenValues HashMap 24 | * 25 | * To make the working set size 1, we can first sort, then iterate 26 | * over the sorted list, emitting only the first occurrence of a 27 | * tuple. We would need to keep only one key in memory, instead of 28 | * the list of all seen values till now. 29 | */ 30 | 31 | private Schema schema; /* Schema for this table */ 32 | 33 | 34 | private Operator child; /* The child operator */ 35 | 36 | private HashSet seenValues; /* Keeps a record 37 | of seen values till now. 38 | The Boolean is a bogus 39 | value, we just need to 40 | check for the presence 41 | of the key in the map. */ 42 | 43 | 44 | public DistinctOperator(Operator child) { 45 | this.child = child; 46 | 47 | /* Schema is unchanged from the child's schema */ 48 | schema = new Schema(child.getSchema()); 49 | 50 | /* Set an appropriate table name, for book-keeping */ 51 | generateSchemaName(); 52 | 53 | /* Initializations */ 54 | seenValues = new HashSet(); 55 | } 56 | 57 | public void generateSchemaName() { 58 | child.generateSchemaName(); 59 | schema.setTableName("\u03B4(" + schema.getTableName() + ")"); 60 | } 61 | 62 | 63 | @Override 64 | public Schema getSchema() { 65 | return schema; 66 | } 67 | 68 | @Override 69 | public void initialize() { 70 | child.initialize(); 71 | 72 | /* Schema is unchanged from the child's schema */ 73 | schema = new Schema(child.getSchema()); 74 | 75 | /* Set an appropriate table name, for book-keeping */ 76 | generateSchemaName(); 77 | } 78 | 79 | @Override 80 | public LeafValue[] readOneTuple() { 81 | /* Necessary initialization */ 82 | LeafValue[] next = null; 83 | 84 | /* Keep getting tuples till we find one we haven't seen yet or we run out */ 85 | while ((next = child.readOneTuple()) != null) { 86 | 87 | /* Generate a key for the tuple, by concatenating each attribute value */ 88 | String key = ""; 89 | for(int i=0; i getDetails() { 140 | Map map = new HashMap(); 141 | List expList = new ArrayList(); 142 | List srcs = new ArrayList(); 143 | srcs.add(child.getDetails()); 144 | map.put("TYPE", "DISTINCT"); 145 | map.put("EXPRESSION", expList); 146 | map.put("SRC", srcs); 147 | return map; 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/GraceHashJoinOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import net.sf.jsqlparser.expression.BinaryExpression; 9 | import net.sf.jsqlparser.expression.Expression; 10 | import net.sf.jsqlparser.expression.LeafValue; 11 | import net.sf.jsqlparser.expression.LeafValue.InvalidLeaf; 12 | import net.sf.jsqlparser.expression.operators.conditional.AndExpression; 13 | import net.sf.jsqlparser.schema.Column; 14 | import edu.buffalo.cse562.ParseTreeOptimizer; 15 | import edu.buffalo.cse562.schema.ColumnWithType; 16 | import edu.buffalo.cse562.schema.Schema; 17 | 18 | public class GraceHashJoinOperator implements Operator { 19 | 20 | /* 21 | * External Hash Join Operator 22 | * Build hashes over both child relations, 23 | * and join them in memory 24 | * 25 | * Constructor Variables 26 | * A where predicate to give the join clause 27 | * Two child operators 28 | * 29 | * Working Set Size - Size of the hash of the smaller table 30 | */ 31 | 32 | private Schema schema; /* Schema for this table */ 33 | 34 | 35 | private Expression where; /* The join clause */ 36 | private Operator child1, child2; /* The two relations to cross product */ 37 | 38 | private int joinedLength; 39 | private ArrayList tempList; /* Temporary list that holds the joined tuples */ 40 | 41 | /* Hashes for the children's keys */ 42 | private HashMap> hash; 43 | 44 | /* Boolean array that contains selected columns for both relations */ 45 | private int selectedCols1; 46 | private int selectedCols2; 47 | private int child1Length; 48 | private int child2Length; 49 | 50 | public GraceHashJoinOperator(Expression where, Operator child1, Operator child2) { 51 | this.where = where; 52 | this.child1 = child1; 53 | this.child2 = child2; 54 | 55 | tempList = new ArrayList(); 56 | hash = new HashMap>(1000000, (float) 0.5); 57 | 58 | buildSchema(); 59 | } 60 | 61 | public void appendWhere(Expression where) { 62 | this.where = new AndExpression(this.where, where); 63 | } 64 | 65 | private void buildSchema() { 66 | 67 | schema = new Schema(); 68 | generateSchemaName(); 69 | 70 | for(ColumnWithType c:child1.getSchema().getColumns()) { 71 | schema.addColumn(c); 72 | } 73 | 74 | for(ColumnWithType c:child2.getSchema().getColumns()) { 75 | schema.addColumn(c); 76 | } 77 | 78 | } 79 | 80 | @Override 81 | public void generateSchemaName() { 82 | child1.generateSchemaName(); 83 | child2.generateSchemaName(); 84 | 85 | schema.setTableName( 86 | "(" + 87 | child1.getSchema().getTableName() + 88 | " \u2A1D" + 89 | " {" + where + "} " + 90 | child2.getSchema().getTableName() + 91 | ")" 92 | ); 93 | 94 | } 95 | 96 | @Override 97 | public Schema getSchema() { 98 | return schema; 99 | } 100 | 101 | @Override 102 | public void initialize() { 103 | /* First initialize the children */ 104 | child1.initialize(); 105 | child2.initialize(); 106 | 107 | child1Length = child1.getSchema().getColumns().size(); 108 | child2Length = child2.getSchema().getColumns().size(); 109 | 110 | /* Initializations */ 111 | selectedCols1 = -1; 112 | selectedCols2 = -1; 113 | joinedLength = child1Length + child2Length; 114 | 115 | /* Get the columns on which to build the key */ 116 | getSelectedColumns(); 117 | 118 | /* Build the hash on both tables */ 119 | try { 120 | buildHash(); 121 | } catch(OutOfMemoryError e) { 122 | e.printStackTrace(); 123 | } 124 | 125 | /* Generate the temporary list of joined tuples */ 126 | buildJoin(); 127 | buildSchema(); 128 | 129 | } 130 | 131 | private void getSelectedColumns() { 132 | 133 | ArrayList clauseList = new ArrayList(); 134 | 135 | if(where instanceof AndExpression) { 136 | clauseList.addAll(ParseTreeOptimizer.splitAndClauses(where)); 137 | } 138 | else { 139 | clauseList.add(where); 140 | } 141 | 142 | for(Expression clause : clauseList) { 143 | BinaryExpression binaryClause = (BinaryExpression) clause; 144 | Column left = (Column) binaryClause.getLeftExpression(); 145 | Column right = (Column) binaryClause.getRightExpression(); 146 | 147 | int pos = findInSchema(left, child1.getSchema()); 148 | if(pos < 0) { 149 | selectedCols2 = findInSchema(left, child2.getSchema()); 150 | selectedCols1 = findInSchema(right, child1.getSchema()); 151 | } 152 | else { 153 | selectedCols1 = pos; 154 | selectedCols2 = findInSchema(right, child2.getSchema()); 155 | } 156 | 157 | } 158 | 159 | } 160 | 161 | private int findInSchema(Column col, Schema schema) { 162 | 163 | ArrayList columnList = schema.getColumns(); 164 | for(int i=0; i toBeAdded = new ArrayList(); 186 | toBeAdded.add(next); 187 | hash.put(key, toBeAdded); 188 | continue; 189 | } 190 | 191 | hash.get(key).add(next); 192 | } 193 | 194 | } 195 | 196 | @Override 197 | public LeafValue[] readOneTuple() { 198 | if(tempList.isEmpty()) { 199 | /* 200 | * no more tuples to return 201 | */ 202 | return null; 203 | } 204 | 205 | /* Temporary return tuple */ 206 | LeafValue[] next = tempList.get(0); 207 | 208 | /* Remove tuple from tempList */ 209 | tempList.remove(0); 210 | 211 | return next; 212 | } 213 | 214 | private void buildJoin(){ 215 | 216 | LeafValue[] next = null; 217 | 218 | while((next = child2.readOneTuple()) != null) { 219 | Long key = null; 220 | try { 221 | key = next[selectedCols2].toLong(); 222 | } catch (InvalidLeaf e) { 223 | e.printStackTrace(); 224 | } 225 | 226 | 227 | ArrayList matchedTuples = hash.get(key); 228 | 229 | if(matchedTuples == null) { 230 | continue; 231 | } 232 | 233 | for(LeafValue[] left : matchedTuples) { 234 | LeafValue[] joinedTuple = new LeafValue[joinedLength]; 235 | for(int i=0; i getDetails() { 287 | Map map = new HashMap(); 288 | List expList = new ArrayList(); 289 | List srcs = new ArrayList(); 290 | srcs.add(child1.getDetails()); 291 | srcs.add(child2.getDetails()); 292 | expList.add(where.toString()); 293 | map.put("TYPE", "JOIN"); 294 | map.put("EXPRESSION", expList); 295 | map.put("SRC", srcs); 296 | return map; 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/LimitOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import net.sf.jsqlparser.expression.LeafValue; 9 | import edu.buffalo.cse562.schema.Schema; 10 | 11 | public class LimitOperator implements Operator { 12 | 13 | /* 14 | * Limit Operator 15 | * Scans over the child operator and 16 | * emits only the first l tuples 17 | * 18 | * Constructor Variables 19 | * l, the number of tuples to emit 20 | * The child operator 21 | * 22 | * Working Set Size - 1 23 | */ 24 | 25 | private Schema schema; /* Schema for this table */ 26 | 27 | 28 | private long limit; /* Limit of tuples to emit */ 29 | private Operator child; /* The child operator */ 30 | 31 | private long count; /* Keeps track of the number of tuples 32 | emitted till now */ 33 | 34 | 35 | public LimitOperator(long limit, Operator child) { 36 | this.limit = limit; 37 | this.child = child; 38 | 39 | /* Schema is unchanged from the child's schema */ 40 | schema = new Schema(child.getSchema()); 41 | 42 | /* Set an appropriate table name, for book-keeping */ 43 | generateSchemaName(); 44 | 45 | /* Initialize the count to 0 */ 46 | count = 0; 47 | } 48 | 49 | public void generateSchemaName() { 50 | child.generateSchemaName(); 51 | schema.setTableName("L(" + child.getSchema().getTableName() + ")"); 52 | } 53 | 54 | @Override 55 | public void initialize() { 56 | child.initialize(); 57 | 58 | /* Schema is unchanged from the child's schema */ 59 | schema = new Schema(child.getSchema()); 60 | 61 | /* Set an appropriate table name, for book-keeping */ 62 | generateSchemaName(); 63 | } 64 | 65 | @Override 66 | public Schema getSchema() { 67 | return schema; 68 | } 69 | 70 | @Override 71 | public LeafValue[] readOneTuple() { 72 | LeafValue[] next = child.readOneTuple(); 73 | if(next == null) { 74 | /* 75 | * No more tuples 76 | */ 77 | return null; 78 | } 79 | 80 | if(count >= limit) { 81 | /* Limit reached */ 82 | return null; 83 | } 84 | else { 85 | /* 86 | * Increment the counter before emitting the tuple 87 | * so that we can keep track of the number of tuples 88 | * emitted till now 89 | */ 90 | count++; 91 | return next; 92 | } 93 | } 94 | 95 | @Override 96 | public void reset() { 97 | /* Reset the count to 0 */ 98 | count = 0; 99 | 100 | /* Now reset the child so that we can begin again */ 101 | child.reset(); 102 | } 103 | 104 | @Override 105 | public Operator getLeft() { 106 | return child; 107 | } 108 | 109 | @Override 110 | public Operator getRight() { 111 | return null; 112 | } 113 | 114 | @Override 115 | public void setLeft(Operator o) { 116 | child = o; 117 | } 118 | 119 | @Override 120 | public void setRight(Operator o) { 121 | 122 | } 123 | 124 | @Override 125 | public Map getDetails() { 126 | Map map = new HashMap(); 127 | List expList = new ArrayList(); 128 | List srcs = new ArrayList(); 129 | srcs.add(child.getDetails()); 130 | expList.add(limit + ""); 131 | map.put("TYPE", "LIMIT"); 132 | map.put("EXPRESSION", expList); 133 | map.put("SRC", srcs); 134 | return map; 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/Operator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.util.Map; 4 | 5 | import net.sf.jsqlparser.expression.LeafValue; 6 | import edu.buffalo.cse562.schema.Schema; 7 | 8 | public interface Operator { 9 | 10 | public Schema getSchema(); 11 | public void generateSchemaName(); 12 | 13 | public void initialize(); 14 | public LeafValue[] readOneTuple(); 15 | public void reset(); 16 | 17 | public Operator getLeft(); 18 | public Operator getRight(); 19 | 20 | public void setLeft(Operator o); 21 | public void setRight(Operator o); 22 | 23 | public Map getDetails(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/OrderByOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import edu.buffalo.cse562.LeafValueComparator; 10 | import edu.buffalo.cse562.schema.Schema; 11 | import net.sf.jsqlparser.expression.LeafValue; 12 | import net.sf.jsqlparser.statement.select.OrderByElement; 13 | 14 | public class OrderByOperator implements Operator { 15 | 16 | /* 17 | * Order By Order 18 | * Sorts the child relation 19 | * 20 | * Constructor Variables 21 | * Order By attributes list 22 | * The child operator 23 | * 24 | * Working Set Size - The size of the relation 25 | */ 26 | 27 | private Schema schema; /* Schema for this table */ 28 | 29 | 30 | private Operator child; /* The child operator to be sorted */ 31 | 32 | /* Holds the sorted relation in memory */ 33 | private ArrayList tempList; 34 | 35 | /* Holds the index into tempList of the next tuple to be emitted */ 36 | private int index; 37 | 38 | 39 | /* Holds the sort key attributes */ 40 | private ArrayList arguments; 41 | 42 | 43 | public OrderByOperator(ArrayList arguments, Operator child) { 44 | this.arguments = arguments; 45 | this.child = child; /* Schema is unchanged from the child's schema */ 46 | 47 | schema = new Schema(child.getSchema()); 48 | 49 | /* Set an appropriate table name, for book-keeping */ 50 | generateSchemaName(); 51 | 52 | /* Initializations */ 53 | tempList = new ArrayList(); 54 | 55 | index = 0; 56 | 57 | } 58 | 59 | public void generateSchemaName() { 60 | child.generateSchemaName(); 61 | schema.setTableName(child.getSchema().getTableName()); 62 | } 63 | 64 | 65 | @Override 66 | public Schema getSchema() { 67 | return schema; 68 | } 69 | 70 | @Override 71 | public void initialize() { 72 | child.initialize(); 73 | 74 | generateSchemaName(); 75 | 76 | /* 77 | * Since this is an in-memory operation, initialize will load the 78 | * entire child relation into memory before sorting and then sort 79 | * it in memory 80 | */ 81 | sort(); 82 | 83 | } 84 | 85 | @Override 86 | public LeafValue[] readOneTuple() { 87 | /* Emit the next tuple in tempList, if any */ 88 | if(index < tempList.size()) { 89 | LeafValue[] ret = tempList.get(index); 90 | index++; 91 | return ret; 92 | } 93 | 94 | /* Reached end of tempList, no more tuples to return */ 95 | return null; 96 | } 97 | 98 | @Override 99 | public void reset() { 100 | /* First set the index to 0 */ 101 | index = 0; 102 | 103 | /* Then reset the child */ 104 | child.reset(); 105 | } 106 | 107 | 108 | private void sort() { 109 | /* The sort function, calls on the sorting routine */ 110 | 111 | /* Load entire child relation into tempList */ 112 | LeafValue[] next = child.readOneTuple(); 113 | while (next != null) { 114 | tempList.add(next); 115 | next = child.readOneTuple(); 116 | } 117 | 118 | /* Sort tempList using the sorting routine */ 119 | Collections.sort(tempList, new LeafValueComparator(arguments, schema)); 120 | } 121 | 122 | @Override 123 | public Operator getLeft() { 124 | return child; 125 | } 126 | 127 | @Override 128 | public Operator getRight() { 129 | return null; 130 | } 131 | 132 | @Override 133 | public void setLeft(Operator o) { 134 | child = o; 135 | } 136 | 137 | @Override 138 | public void setRight(Operator o) { 139 | 140 | } 141 | 142 | public ArrayList getArguments() { 143 | return arguments; 144 | } 145 | 146 | @Override 147 | public Map getDetails() { 148 | Map map = new HashMap(); 149 | List expList = new ArrayList(); 150 | for(OrderByElement arg : arguments) 151 | expList.add(arg.toString()); 152 | List srcs = new ArrayList(); 153 | srcs.add(child.getDetails()); 154 | map.put("TYPE", "ORDERBY"); 155 | map.put("EXPRESSION", expList); 156 | map.put("SRC", srcs); 157 | return map; 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/ProjectScanOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileNotFoundException; 6 | import java.io.FileReader; 7 | import java.io.IOException; 8 | import java.util.ArrayList; 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.HashSet; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | import net.sf.jsqlparser.expression.DoubleValue; 16 | import net.sf.jsqlparser.expression.LeafValue; 17 | import net.sf.jsqlparser.expression.LongValue; 18 | import net.sf.jsqlparser.expression.StringValue; 19 | import edu.buffalo.cse562.schema.ColumnWithType; 20 | import edu.buffalo.cse562.schema.Schema; 21 | 22 | public class ProjectScanOperator implements Operator { 23 | 24 | private Schema oldSchema; 25 | private Schema newSchema; 26 | 27 | 28 | private HashSet selectedColumnNames; 29 | private boolean[] selectedCols; 30 | 31 | 32 | private BufferedReader br; 33 | private File f; 34 | private FileReader fr; 35 | 36 | 37 | 38 | 39 | public ProjectScanOperator(Schema schema, HashSet selectedColumnNames) { 40 | this.oldSchema = schema; 41 | this.selectedColumnNames = selectedColumnNames; 42 | 43 | 44 | f = new File(schema.getTableFile()); 45 | 46 | selectedCols = new boolean[schema.getColumns().size()]; 47 | Arrays.fill(selectedCols, false); 48 | 49 | buildSchema(); 50 | } 51 | 52 | private void buildSchema() { 53 | newSchema = new Schema(oldSchema); 54 | newSchema.clearColumns(); 55 | 56 | int i = 0; 57 | for(ColumnWithType c : oldSchema.getColumns()) { 58 | if(selectedColumnNames.contains(c.getColumnName().toLowerCase())) { 59 | newSchema.addColumn(c); 60 | selectedCols[i] = true; 61 | } 62 | 63 | i++; 64 | } 65 | } 66 | 67 | @Override 68 | public Schema getSchema() { 69 | return newSchema; 70 | } 71 | 72 | @Override 73 | public void generateSchemaName() { 74 | 75 | } 76 | 77 | @Override 78 | public void initialize() { 79 | 80 | buildSchema(); 81 | 82 | try { 83 | fr = new FileReader(f); 84 | } catch (FileNotFoundException e) { 85 | e.printStackTrace(); 86 | } 87 | br = new BufferedReader(fr); 88 | 89 | 90 | } 91 | 92 | @Override 93 | public LeafValue[] readOneTuple() { 94 | if(br == null) { 95 | /* 96 | * May happen if the operator is not 97 | * initialized before use 98 | */ 99 | return null; 100 | } 101 | 102 | String line = null; 103 | try { 104 | /* Read the next '|' delimited line which is the 105 | * next tuple 106 | */ 107 | line = br.readLine(); 108 | } catch (IOException e) { 109 | e.printStackTrace(); 110 | } 111 | 112 | if(line == null) { 113 | /* 114 | * No more tuples to be read 115 | */ 116 | return null; 117 | } 118 | 119 | return constructTuple(line); 120 | 121 | 122 | } 123 | 124 | public LeafValue[] constructTuple(String line) { 125 | /* Split the tuple into attributes using the '|' delimiter */ 126 | String cols[] = line.split("\\|"); 127 | 128 | /* LeafValue array that will hold the tuple to be returned */ 129 | LeafValue ret[] = new LeafValue[newSchema.getColumns().size()]; 130 | 131 | /* 132 | * Iterate over each column according to schema's type information 133 | * and populate the ret array with appropriate value and LeafValue 134 | * type 135 | */ 136 | int k = 0; 137 | for(int i=0; i getSelectedColumns() { 210 | return selectedColumnNames; 211 | } 212 | 213 | @Override 214 | public Map getDetails() { 215 | Map map = new HashMap(); 216 | List expList = new ArrayList(); 217 | expList.add(oldSchema.getTableName() + "=" + oldSchema.getTableFile()); 218 | for(ColumnWithType col : oldSchema.getColumns()) 219 | expList.add(col.getColumnName() + "=" + col.getColumnType()); 220 | List srcs = new ArrayList(); 221 | map.put("TYPE", "TABLE"); 222 | map.put("EXPRESSION", expList); 223 | map.put("SRC", srcs); 224 | return map; 225 | } 226 | 227 | } 228 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/ScanOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileNotFoundException; 6 | import java.io.FileReader; 7 | import java.io.IOException; 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import net.sf.jsqlparser.expression.DateValue; 14 | import net.sf.jsqlparser.expression.DoubleValue; 15 | import net.sf.jsqlparser.expression.LeafValue; 16 | import net.sf.jsqlparser.expression.LongValue; 17 | import net.sf.jsqlparser.expression.StringValue; 18 | import edu.buffalo.cse562.schema.ColumnWithType; 19 | import edu.buffalo.cse562.schema.Schema; 20 | 21 | public class ScanOperator implements Operator { 22 | 23 | /* 24 | * Scan Operator 25 | * Scans over a file, known from the schema 26 | * and emits a tuple as a LeafValue array 27 | * 28 | * Constructor Variables 29 | * A schema, representing the relation 30 | * to be scanned 31 | * 32 | * Working Set Size - 1 33 | */ 34 | 35 | private Schema schema; 36 | 37 | 38 | private BufferedReader br; 39 | private File f; 40 | private FileReader fr; 41 | 42 | 43 | private String line; 44 | 45 | public ScanOperator(Schema schema) { 46 | /* 47 | * Get the initial schema, 48 | * the schemas for the rest of the operators 49 | * are ultimately generated 50 | * using this information 51 | */ 52 | this.schema = new Schema(schema); 53 | /* 54 | * Make a new File object using the file location 55 | * information in the relation's schema. The 56 | * initial schema is generated during parsetree 57 | * creation 58 | */ 59 | f = new File(schema.getTableFile()); 60 | 61 | line = null; 62 | } 63 | 64 | public void generateSchemaName() { 65 | 66 | } 67 | 68 | @Override 69 | public Schema getSchema() { 70 | return schema; 71 | } 72 | 73 | @Override 74 | public void initialize() { 75 | 76 | try { 77 | fr = new FileReader(f); 78 | } catch (FileNotFoundException e) { 79 | e.printStackTrace(); 80 | } 81 | br = new BufferedReader(fr); 82 | } 83 | 84 | @Override 85 | public LeafValue[] readOneTuple() { 86 | if(br == null) { 87 | /* 88 | * May happen if the operator is not 89 | * initialized before use 90 | */ 91 | return null; 92 | } 93 | 94 | try { 95 | /* Read the next '|' delimited line which is the 96 | * next tuple 97 | */ 98 | line = br.readLine(); 99 | } catch (IOException e) { 100 | e.printStackTrace(); 101 | } 102 | 103 | if(line == null) { 104 | /* 105 | * No more tuples to be read 106 | */ 107 | return null; 108 | } 109 | 110 | /* Split the tuple into attributes using the '|' delimiter */ 111 | String cols[] = line.split("\\|"); 112 | 113 | /* LeafValue array that will hold the tuple to be returned */ 114 | LeafValue ret[] = new LeafValue[cols.length]; 115 | 116 | /* 117 | * Iterate over each column according to schema's type information 118 | * and populate the ret array with appropriate value and LeafValue 119 | * type 120 | */ 121 | for(int i=0; i getDetails() { 197 | Map map = new HashMap(); 198 | List expList = new ArrayList(); 199 | expList.add(schema.getTableName() + "=" + schema.getTableFile()); 200 | for(ColumnWithType col : schema.getColumns()) 201 | expList.add(col.getColumnName() + "=" + col.getColumnType()); 202 | List srcs = new ArrayList(); 203 | map.put("TYPE", "TABLE"); 204 | map.put("EXPRESSION", expList); 205 | map.put("SRC", srcs); 206 | return map; 207 | } 208 | 209 | } -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/SelectionOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.sql.SQLException; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import net.sf.jsqlparser.expression.BooleanValue; 10 | import net.sf.jsqlparser.expression.DateValue; 11 | import net.sf.jsqlparser.expression.DoubleValue; 12 | import net.sf.jsqlparser.expression.Expression; 13 | import net.sf.jsqlparser.expression.LeafValue; 14 | import net.sf.jsqlparser.expression.LongValue; 15 | import net.sf.jsqlparser.expression.StringValue; 16 | import net.sf.jsqlparser.schema.Column; 17 | import edu.buffalo.cse562.Eval; 18 | import edu.buffalo.cse562.ParseTreeOptimizer; 19 | import edu.buffalo.cse562.schema.ColumnInfo; 20 | import edu.buffalo.cse562.schema.Schema; 21 | 22 | public class SelectionOperator extends Eval implements Operator { 23 | 24 | /* 25 | * Selection Operator 26 | * Scans over the child operator and 27 | * emits only the tuples matching 28 | * the selection predicate represented 29 | * by the where Expression 30 | * 31 | * Constructor Variables 32 | * The where clause 33 | * The child operator 34 | * 35 | * Working Set Size - 1 36 | */ 37 | 38 | private Schema schema; /* Schema for this table */ 39 | 40 | 41 | private Expression where; /* Selection predicates */ 42 | private Operator child; /* Child operator */ 43 | 44 | /* 45 | * next[] needs to be a class variable, because eval() 46 | * needs access to it 47 | */ 48 | private LeafValue next[]; 49 | 50 | /* 51 | * 52 | * TypeCache is a HashMap that enables 53 | * optimization for the eval function, 54 | * this cache enables constant time access 55 | * to the type information for a Column, 56 | * we do not need to iterate over all columns 57 | * to find the type of the column 58 | * for every single tuple, 59 | * once we have built the cache while 60 | * processing the first tuple, that information 61 | * can be looked up in this Map. 62 | * 63 | */ 64 | private HashMap TypeCache; 65 | 66 | private ArrayList clauseList; 67 | 68 | 69 | 70 | public SelectionOperator(Expression where, Operator child) { 71 | this.where = where; 72 | this.child = child; 73 | 74 | /* Schema is unchanged from the child's schema */ 75 | schema = new Schema(child.getSchema()); 76 | 77 | /* Set an appropriate table name, for book-keeping */ 78 | generateSchemaName(); 79 | 80 | /* Initializations */ 81 | TypeCache = new HashMap(); 82 | } 83 | 84 | public void generateSchemaName() { 85 | child.generateSchemaName(); 86 | schema.setTableName("\u03C3 {" + where + "}(" + child.getSchema().getTableName() + ")"); 87 | } 88 | 89 | @Override 90 | public Schema getSchema() { 91 | return schema; 92 | } 93 | 94 | 95 | @Override 96 | public void initialize() { 97 | child.initialize(); 98 | schema = child.getSchema(); 99 | generateSchemaName(); 100 | clauseList = ParseTreeOptimizer.splitAndClauses(where); 101 | } 102 | 103 | @Override 104 | public LeafValue[] readOneTuple() { 105 | 106 | /* 107 | * Keep iterating over tuples till there are no more tuples 108 | */ 109 | while((next = child.readOneTuple()) != null) { 110 | BooleanValue test = null; 111 | try { 112 | /* 113 | * Test the validity of the selection predicate 114 | * for the current tuple, represented by next[] 115 | */ 116 | for(int i=0; i getDetails() { 242 | Map map = new HashMap(); 243 | List expList = new ArrayList(); 244 | List srcs = new ArrayList(); 245 | srcs.add(child.getDetails()); 246 | expList.add(where.toString()); 247 | map.put("TYPE", "SELECT"); 248 | map.put("EXPRESSION", expList); 249 | map.put("SRC", srcs); 250 | return map; 251 | } 252 | 253 | } -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/operators/UnionOperator.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.operators; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import net.sf.jsqlparser.expression.LeafValue; 9 | import edu.buffalo.cse562.schema.Schema; 10 | 11 | public class UnionOperator implements Operator { 12 | 13 | /* 14 | * Union Operator 15 | * Scans over the first child operator, 16 | * then scans over the second one 17 | * 18 | * Constructor Variables 19 | * Two child operators 20 | * 21 | * Working Set Size - 1 22 | */ 23 | 24 | private Schema schema; /* Schema for this table */ 25 | 26 | 27 | private Operator child1, child2; /* The two relations to the union */ 28 | 29 | private boolean child1Done; /* A flag to keep track of 30 | whether child1 is done or not */ 31 | 32 | public UnionOperator(Operator child1, Operator child2) { 33 | this.child1 = child1; 34 | this.child2 = child2; 35 | 36 | /* Schema is unchanged from the child's schema */ 37 | schema = new Schema(child1.getSchema()); 38 | 39 | /* Set an appropriate table name, for book-keeping */ 40 | generateSchemaName(); 41 | 42 | /* Initially the flag is off course false */ 43 | child1Done = false; 44 | } 45 | 46 | 47 | @Override 48 | public void initialize() { 49 | child1.initialize(); 50 | child2.initialize(); 51 | 52 | /* Schema is unchanged from the child's schema */ 53 | schema = new Schema(child1.getSchema()); 54 | 55 | /* Set an appropriate table name, for book-keeping */ 56 | generateSchemaName(); 57 | } 58 | 59 | 60 | @Override 61 | public void generateSchemaName() { 62 | child1.generateSchemaName(); 63 | child2.generateSchemaName(); 64 | 65 | schema.setTableName("(" + child1.getSchema().getTableName() + " U " + child2.getSchema().getTableName() + ")"); 66 | } 67 | 68 | @Override 69 | public Schema getSchema() { 70 | return schema; 71 | } 72 | 73 | @Override 74 | public LeafValue[] readOneTuple() { 75 | 76 | if(!child1Done) { 77 | LeafValue[] next = child1.readOneTuple(); 78 | if(next == null) { 79 | child1Done = true; 80 | 81 | /* 82 | * Safe to do this, because the recursion stack will 83 | * grow by only one call, however big the input 84 | */ 85 | return readOneTuple(); 86 | } 87 | 88 | return next; 89 | } 90 | 91 | /* 92 | * No need to check for null, since the return will anyway be 93 | * null if the next value is null 94 | */ 95 | return child2.readOneTuple(); 96 | 97 | } 98 | 99 | @Override 100 | public void reset() { 101 | /* First reset the flag */ 102 | child1Done = false; 103 | 104 | /* Then reset the children */ 105 | child1.reset(); 106 | child2.reset(); 107 | } 108 | 109 | @Override 110 | public Operator getLeft() { 111 | return child1; 112 | } 113 | 114 | @Override 115 | public Operator getRight() { 116 | return child2; 117 | } 118 | 119 | @Override 120 | public void setLeft(Operator o) { 121 | child1 = o; 122 | } 123 | 124 | @Override 125 | public void setRight(Operator o) { 126 | child2 = o; 127 | } 128 | 129 | 130 | @Override 131 | public Map getDetails() { 132 | Map map = new HashMap(); 133 | List expList = new ArrayList(); 134 | List srcs = new ArrayList(); 135 | srcs.add(child1.getDetails()); 136 | srcs.add(child2.getDetails()); 137 | map.put("TYPE", "UNION"); 138 | map.put("EXPRESSION", expList); 139 | map.put("SRC", srcs); 140 | return map; 141 | } 142 | 143 | } -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/schema/ColumnInfo.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.schema; 2 | 3 | public class ColumnInfo { 4 | 5 | /* 6 | * A helper class to contain information necessary for the TypeCache 7 | * optimizations in several Operators 8 | */ 9 | 10 | public String type; 11 | public int pos; 12 | 13 | 14 | public ColumnInfo(String type, int pos) { 15 | this.type = type; 16 | this.pos = pos; 17 | } 18 | 19 | } -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/schema/ColumnWithType.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.schema; 2 | 3 | import net.sf.jsqlparser.schema.Column; 4 | import net.sf.jsqlparser.schema.Table; 5 | 6 | public class ColumnWithType extends Column { 7 | 8 | /* 9 | * An augmentation of the Column class to include 10 | * type information and the attribute order information 11 | */ 12 | 13 | private String columnType; 14 | private int columnNumber; 15 | 16 | 17 | public ColumnWithType(Table table, String columnName, String columnType, int columnNumber) { 18 | super(table, columnName); 19 | this.columnType = columnType; 20 | this.columnNumber = columnNumber; 21 | } 22 | 23 | public ColumnWithType(Column col, String columnType, int columnNumber) { 24 | super(col.getTable(), col.getColumnName()); 25 | this.columnType = columnType; 26 | this.columnNumber = columnNumber; 27 | } 28 | 29 | public ColumnWithType(ColumnWithType col) { 30 | super(col.getTable(), col.getColumnName()); 31 | this.columnType = col.columnType; 32 | this.columnNumber = col.columnNumber; 33 | } 34 | 35 | /* Getters and Setters */ 36 | public String getColumnType() { 37 | return columnType; 38 | } 39 | 40 | public void setColumnType(String columnType) { 41 | this.columnType = columnType; 42 | } 43 | 44 | public int getColumnNumber() { 45 | return columnNumber; 46 | } 47 | 48 | public void setColumnNumber(int columnNumber) { 49 | this.columnNumber = columnNumber; 50 | } 51 | 52 | public Column getColumn() { 53 | return new Column(this.getTable(), this.getColumnName()); 54 | } 55 | 56 | 57 | 58 | @Override 59 | public String toString() { 60 | return this.getColumn().getWholeColumnName(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /MushroomCloud/src/edu/buffalo/cse562/schema/Schema.java: -------------------------------------------------------------------------------- 1 | package edu.buffalo.cse562.schema; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | 7 | import net.sf.jsqlparser.schema.Column; 8 | 9 | public class Schema { 10 | 11 | /* 12 | * A class to encapsulate the Schema information of a relation. 13 | * 14 | * tableName - The name of the table 15 | * tableFile - In case the relation is a table on disk, this attribute 16 | * contains the location and name of that file, otherwise 17 | * it is set to "__mem__" indicating that it is a relation 18 | * in memory 19 | * 20 | * columns - A list of the columns of this relation 21 | */ 22 | 23 | private String tableName; 24 | private String tableFile; 25 | 26 | private ArrayList columns; 27 | 28 | private long rowCount; 29 | 30 | 31 | public Schema(String tableName, String tableFile) { 32 | this.tableName = tableName; 33 | this.tableFile = tableFile; 34 | 35 | rowCount = 0; 36 | 37 | /* Initializations */ 38 | columns = new ArrayList(); 39 | } 40 | 41 | public Schema(Schema schema) { 42 | this.tableName = schema.tableName; 43 | this.tableFile = schema.tableFile; 44 | this.columns = new ArrayList(); 45 | 46 | rowCount = schema.getRowCount(); 47 | 48 | for(int i=0; i(); 62 | } 63 | 64 | 65 | 66 | /* Getters and Setters */ 67 | public String getTableName() { 68 | return tableName; 69 | } 70 | 71 | public void setTableName(String tableName) { 72 | this.tableName = tableName; 73 | } 74 | 75 | public String getTableFile() { 76 | return tableFile; 77 | } 78 | 79 | public void setTableFile(String tableFile) { 80 | this.tableFile = tableFile; 81 | } 82 | 83 | public ArrayList getColumns() { 84 | return columns; 85 | } 86 | 87 | public void addColumn(ColumnWithType column) { 88 | this.columns.add(column); 89 | } 90 | 91 | public void addColumns(ColumnWithType columns[]) { 92 | this.columns.addAll(Arrays.asList(columns)); 93 | } 94 | 95 | public int columnToIndex(ColumnWithType col) { 96 | for(int i=0; i ") 8 | if(query == "q" or query == "quit"): 9 | break 10 | 11 | with open('./bin/t.sql', 'w') as f: 12 | f.write(query) 13 | 14 | mushroomCloud = subprocess.Popen( 15 | ['java', '-jar', './bin/MushroomCloud.jar', 16 | '--data', './test/data', 17 | './test/sql/tpch_schemas.sql', './bin/t.sql'], 18 | stdout=subprocess.PIPE 19 | ) 20 | 21 | json = mushroomCloud.stdout.readline() 22 | valkyrie = subprocess.Popen(['./bin/llvmruntime'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) 23 | result = valkyrie.communicate(input=json) 24 | print result[0] 25 | 26 | if __name__ == "__main__": 27 | promptLoop() -------------------------------------------------------------------------------- /Valkyrie/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_COMPILER clang++-3.5) 2 | 3 | set(CMD_LLVM_INC llvm-config-3.5 --system-libs --libs core native jit bitwriter) 4 | set(CMD_FLAGS llvm-config-3.5 --cxxflags --ldflags) 5 | 6 | 7 | execute_process(COMMAND ${CMD_LLVM_INC} OUTPUT_VARIABLE LLVM_LIBS OUTPUT_STRIP_TRAILING_WHITESPACE) 8 | execute_process(COMMAND ${CMD_FLAGS} OUTPUT_VARIABLE FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) 9 | 10 | set(CMAKE_CXX_FLAGS ${FLAGS}) 11 | 12 | 13 | file(GLOB VALKYRIE_INC "include/*.h") 14 | file(GLOB VALKYRIE_SRC "src/*.cpp") 15 | 16 | include_directories(PRIVATE include PRIVATE ${LLVM_LIBS}) 17 | 18 | set(SOURCE_FILES ${VALKYRIE_INC} ${VALKYRIE_SRC}) 19 | add_executable(VALKYRIE ${SOURCE_FILES}) 20 | -------------------------------------------------------------------------------- /Valkyrie/Makefile: -------------------------------------------------------------------------------- 1 | INC_DIR = ./include 2 | SRC_DIR = ./src 3 | OBJ_DIR = ./object 4 | 5 | SRC_FILES = $(wildcard $(SRC_DIR)/*.cpp) 6 | OBJ_FILES = $(SRC_FILES:.cpp=.o) 7 | OBJ_PATH = $(patsubst $(SRC_DIR)/%,$(OBJ_DIR)/%,$(OBJ_FILES)) 8 | 9 | LIBS= 10 | CC=clang++-3.5 11 | CFLAGS=-g -O3 `llvm-config-3.5 --cxxflags --ldflags --system-libs --libs core mcjit native bitwriter` 12 | 13 | all: valkyrie 14 | 15 | $(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp 16 | $(CC) -c -o $@ $< $(CFLAGS) 17 | 18 | valkyrie: 19 | mkdir -p build 20 | $(CC) $(SRC_DIR)/*.cpp $(CFLAGS) -o build/llvmruntime -I$(INC_DIR) 21 | 22 | clean: 23 | rm -f $(OBJ_DIR)/*.o $(INC_DIR)/*~ valkyrie 24 | rm -rf build -------------------------------------------------------------------------------- /Valkyrie/include/Codegen.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef CDB_CODEGEN_H 3 | #define CDB_CODEGEN_H 4 | 5 | #include "llvm/IR/LLVMContext.h" 6 | #include "llvm/IR/Module.h" 7 | #include "llvm/IR/IRBuilder.h" 8 | #include "llvm/ExecutionEngine/ExecutionEngine.h" 9 | 10 | #include "DataTypes.h" 11 | #include "Operator.h" 12 | #include "Schema.h" 13 | #include "Expression.h" 14 | #include "JoinOperator.h" 15 | 16 | 17 | typedef enum { 18 | GET_PTR, 19 | GET_SIZE, 20 | LD_BLK, 21 | CLOSE 22 | } SCHEMA_OP; 23 | 24 | 25 | using namespace llvm; 26 | using namespace std; 27 | using namespace valkyrie; 28 | 29 | namespace codegen { 30 | 31 | void initialize(std::string); 32 | ExecutionEngine* compile(); 33 | 34 | IRBuilder<>* getBuilder(); 35 | Value* getTupleptr(); 36 | size_t getAttPos(string colname); 37 | DataType getAttType(string colname); 38 | DataType getAttType(int colpos); 39 | void scanConsume(Schema&, valkyrie::Operator* parent); 40 | void selectConsume(Expression* expressions, valkyrie::Operator *parent); 41 | void printConsume(int*); 42 | void joinConsume(Schema& schema, valkyrie::Operator *parent); 43 | void joinLeftConsume(JoinOperator*); 44 | void joinRightConsume(JoinOperator*); 45 | void setSchema(Schema *schema); 46 | Schema* getSchema(); 47 | 48 | Value* stringCmp(Value*, Value*, ExprType); 49 | 50 | void nollvm(); 51 | } 52 | 53 | #endif //CDB_CODEGEN_H 54 | -------------------------------------------------------------------------------- /Valkyrie/include/DataTypes.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECT_DATATYPES_H 2 | #define PROJECT_DATATYPES_H 3 | 4 | #include 5 | 6 | using namespace std; 7 | 8 | enum DataType { 9 | LONG, 10 | DOUBLE, 11 | STRING, 12 | DATE 13 | }; 14 | 15 | typedef union { 16 | long l; 17 | double d; 18 | char* c; 19 | } LeafValue; 20 | 21 | typedef struct tuplePointer { 22 | int64_t ptr; 23 | size_t att_count; 24 | size_t tup_count; 25 | } TupPtr; 26 | 27 | #endif //PROJECT_DATATYPES_H 28 | -------------------------------------------------------------------------------- /Valkyrie/include/ExpressionParser.h: -------------------------------------------------------------------------------- 1 | #ifndef EXPRESSION_PARSER_H 2 | #define EXPRESSION_PARSER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "Expression.h" 8 | #include "DataTypes.h" 9 | 10 | namespace valkyrie{ 11 | class ExpressionParser { 12 | private: 13 | std::unordered_set logicalOperators, arithmeticOperators, relationalOperators; 14 | std::vector splitString(const std::string s, char delimiter); 15 | bool isDouble(std::string s); 16 | bool isLong(std::string s); 17 | bool isDate(std::string s); 18 | bool isString(std::string s); 19 | long extractLong(std::string s); 20 | double extractDouble(std::string s); 21 | valkyrie::Expression* leafExpression(std::string s); 22 | valkyrie::BinaryExpression* binaryExpression(std::string s); 23 | valkyrie::Expression* parseExpression(std::vector s); 24 | public: 25 | ExpressionParser(); 26 | std::vector parse(std::string exp); 27 | }; 28 | } 29 | 30 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/JoinOperator.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by vinayak on 12/7/15. 3 | // 4 | 5 | #ifndef JOIN_OPERATOR_H 6 | #define JOIN_OPERATOR_H 7 | 8 | #include "Operator.h" 9 | #include "Expression.h" 10 | 11 | namespace valkyrie { 12 | class JoinOperator : public Operator { 13 | private: 14 | Expression* joinClause; 15 | vector left; 16 | vector right; 17 | char status = 0; 18 | 19 | void updateExpression(Expression* exp, unordered_map lm, unordered_maprm 20 | , string ltable, string rtable); 21 | Schema* mergeSchemas(Schema* lsch, Schema* rsch); 22 | public: 23 | unordered_map>> hashtable; 24 | JoinOperator(std::vector expressions, std::vector children); 25 | void produce(); 26 | void consume(); 27 | vector getLeftJoinAttrs(); 28 | vector getRightJoinAttrs(); 29 | }; 30 | } 31 | 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /Valkyrie/include/Operator.h: -------------------------------------------------------------------------------- 1 | #ifndef OPERATOR_H 2 | #define OPERATOR_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "DataTypes.h" 11 | #include "Schema.h" 12 | 13 | namespace valkyrie{ 14 | 15 | class Operator { 16 | protected: 17 | std::string type; 18 | valkyrie::Schema *schema; 19 | std::vector expressions; 20 | Operator* parent; 21 | std::vector children; 22 | DataType mapType(std::string type); 23 | 24 | public: 25 | Operator(std::vector children); 26 | ~Operator(); 27 | valkyrie::Schema* getSchema(); 28 | void setParent(Operator* parent); 29 | std::string queryPlan(); 30 | std::string toString(); 31 | std::vector getChildren(); 32 | 33 | //LLVM code generation 34 | virtual void consume() = 0; 35 | virtual void produce() = 0; 36 | }; 37 | } 38 | 39 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/Parser.h: -------------------------------------------------------------------------------- 1 | #ifndef PARSER_H 2 | #define PARSER_H 3 | 4 | #include 5 | #include "Operator.h" 6 | #include "PrintOperator.h" 7 | #include "ScanOperator.h" 8 | #include "SelectOperator.h" 9 | #include "ProjectionOperator.h" 10 | #include "JoinOperator.h" 11 | #include "rapidjson/document.h" 12 | 13 | namespace valkyrie{ 14 | class Parser { 15 | private: 16 | Operator* createTree(const rapidjson::Value& node); 17 | public: 18 | Operator* parseJson(std::string json); 19 | }; 20 | } 21 | 22 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/PrintOperator.h: -------------------------------------------------------------------------------- 1 | #ifndef PRINT_OPERATOR_H 2 | #define PRINT_OPERATOR_H 3 | 4 | #include "Operator.h" 5 | 6 | namespace valkyrie{ 7 | class PrintOperator : public Operator { 8 | int* types; 9 | public: 10 | PrintOperator(std::vector expressions, std::vector children); 11 | void consume(); 12 | void produce(); 13 | }; 14 | } 15 | 16 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/ProjectionOperator.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTION_OPERATOR_H 2 | #define PROJECTION_OPERATOR_H 3 | 4 | #include 5 | #include "Operator.h" 6 | #include "Expression.h" 7 | 8 | namespace valkyrie{ 9 | class ProjectionOperator : public Operator { 10 | private: 11 | std::vector projectionClauses; 12 | void updateSchema(); 13 | void updateExpression(valkyrie::Expression* newExp, unordered_map m, string tableName); 14 | public: 15 | ProjectionOperator(std::vector expressions, std::vector children); 16 | void consume(); 17 | void produce(); 18 | }; 19 | } 20 | 21 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/ScanOperator.h: -------------------------------------------------------------------------------- 1 | #ifndef SCAN_OPERATOR_H 2 | #define SCAN_OPERATOR_H 3 | 4 | #include "Operator.h" 5 | 6 | namespace valkyrie{ 7 | class ScanOperator : public Operator { 8 | private: 9 | void convertToSchema(); 10 | public: 11 | ScanOperator(std::vector expressions, std::vector children); 12 | ~ScanOperator(); 13 | void consume(); 14 | void produce(); 15 | }; 16 | } 17 | 18 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/Schema.h: -------------------------------------------------------------------------------- 1 | #ifndef CDB_SCHEMA_H 2 | #define CDB_SCHEMA_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "DataTypes.h" 9 | #include "Expression.h" 10 | 11 | #define VDB_BLOCK_SIZE 100000 12 | 13 | using namespace std; 14 | 15 | namespace valkyrie{ 16 | class Schema { 17 | private: 18 | string tablename; 19 | string datafile; 20 | vector attributes; 21 | vector types; 22 | 23 | vector tuples; 24 | unordered_map colMap; 25 | 26 | string formatAttrName(string name) const; 27 | ifstream* infile; 28 | bool block_allocated = true; 29 | 30 | public: 31 | // Constructors 32 | Schema(string tablename); 33 | Schema(string tablename, string datafile); 34 | 35 | // LLVM Controller interface 36 | void init(); 37 | uint64_t loadBlock(); 38 | uint64_t close(); 39 | 40 | // Setup 41 | void addAttribute(string, DataType); 42 | void addAttribute(string, DataType, Expression*); 43 | void addTuple(LeafValue *); 44 | 45 | // Getters, Setters 46 | void setTableName(string tablename); 47 | string getTableName() const; 48 | 49 | uint64_t getTuplePtr() const; 50 | size_t getTupleCount() const; 51 | 52 | void setTuples(vector ts); 53 | vector getTuples(); 54 | 55 | const vector getTypes() const; 56 | 57 | std::string getDataFile() const; 58 | 59 | vector getAttributes() const; 60 | size_t getAttributePos(string) const; 61 | DataType getAttributeType(string) const; 62 | Expression* getAttrExpression(string colName); 63 | unordered_map getColumnMap(); 64 | 65 | 66 | // Utilities 67 | std::string attrsVecsToCommaSepString(const vector& attr, const vector& types) const; 68 | friend ostream& operator<<(ostream &stream, const valkyrie::Schema &schema); 69 | }; 70 | } 71 | 72 | 73 | 74 | #endif //CDB_SCHEMA_H 75 | -------------------------------------------------------------------------------- /Valkyrie/include/SelectOperator.h: -------------------------------------------------------------------------------- 1 | #ifndef SELECT_OPERATOR_H 2 | #define SELECT_OPERATOR_H 3 | 4 | #include "Operator.h" 5 | #include "Expression.h" 6 | #include "ExpressionParser.h" 7 | 8 | namespace valkyrie{ 9 | class SelectOperator : public Operator { 10 | Expression* selectClause; 11 | void updateExpression(Expression *newExp, unordered_map m, std::string tableName); 12 | public: 13 | SelectOperator(std::vector expressions, std::vector children); 14 | void consume(); 15 | void produce(); 16 | }; 17 | } 18 | 19 | #endif -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/error/en.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_ERROR_EN_H__ 16 | #define RAPIDJSON_ERROR_EN_H__ 17 | 18 | #include "error.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | 22 | //! Maps error code of parsing into error message. 23 | /*! 24 | \ingroup RAPIDJSON_ERRORS 25 | \param parseErrorCode Error code obtained in parsing. 26 | \return the error message. 27 | \note User can make a copy of this function for localization. 28 | Using switch-case is safer for future modification of error codes. 29 | */ 30 | inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { 31 | switch (parseErrorCode) { 32 | case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); 33 | 34 | case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); 35 | case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); 36 | 37 | case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); 38 | 39 | case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); 40 | case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); 41 | case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); 42 | 43 | case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); 44 | 45 | case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); 46 | case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); 47 | case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); 48 | case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); 49 | case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); 50 | 51 | case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); 52 | case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); 53 | case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); 54 | 55 | case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); 56 | case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); 57 | 58 | default: 59 | return RAPIDJSON_ERROR_STRING("Unknown error."); 60 | } 61 | } 62 | 63 | RAPIDJSON_NAMESPACE_END 64 | 65 | #endif // RAPIDJSON_ERROR_EN_H__ 66 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/error/error.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_ERROR_ERROR_H__ 16 | #define RAPIDJSON_ERROR_ERROR_H__ 17 | 18 | #include "../rapidjson.h" 19 | 20 | /*! \file error.h */ 21 | 22 | /*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ 23 | 24 | /////////////////////////////////////////////////////////////////////////////// 25 | // RAPIDJSON_ERROR_CHARTYPE 26 | 27 | //! Character type of error messages. 28 | /*! \ingroup RAPIDJSON_ERRORS 29 | The default character type is \c char. 30 | On Windows, user can define this macro as \c TCHAR for supporting both 31 | unicode/non-unicode settings. 32 | */ 33 | #ifndef RAPIDJSON_ERROR_CHARTYPE 34 | #define RAPIDJSON_ERROR_CHARTYPE char 35 | #endif 36 | 37 | /////////////////////////////////////////////////////////////////////////////// 38 | // RAPIDJSON_ERROR_STRING 39 | 40 | //! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. 41 | /*! \ingroup RAPIDJSON_ERRORS 42 | By default this conversion macro does nothing. 43 | On Windows, user can define this macro as \c _T(x) for supporting both 44 | unicode/non-unicode settings. 45 | */ 46 | #ifndef RAPIDJSON_ERROR_STRING 47 | #define RAPIDJSON_ERROR_STRING(x) x 48 | #endif 49 | 50 | RAPIDJSON_NAMESPACE_BEGIN 51 | 52 | /////////////////////////////////////////////////////////////////////////////// 53 | // ParseErrorCode 54 | 55 | //! Error code of parsing. 56 | /*! \ingroup RAPIDJSON_ERRORS 57 | \see GenericReader::Parse, GenericReader::GetParseErrorCode 58 | */ 59 | enum ParseErrorCode { 60 | kParseErrorNone = 0, //!< No error. 61 | 62 | kParseErrorDocumentEmpty, //!< The document is empty. 63 | kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. 64 | 65 | kParseErrorValueInvalid, //!< Invalid value. 66 | 67 | kParseErrorObjectMissName, //!< Missing a name for object member. 68 | kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. 69 | kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. 70 | 71 | kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. 72 | 73 | kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. 74 | kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. 75 | kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. 76 | kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. 77 | kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. 78 | 79 | kParseErrorNumberTooBig, //!< Number too big to be stored in double. 80 | kParseErrorNumberMissFraction, //!< Miss fraction part in number. 81 | kParseErrorNumberMissExponent, //!< Miss exponent in number. 82 | 83 | kParseErrorTermination, //!< Parsing was terminated. 84 | kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. 85 | }; 86 | 87 | //! Result of parsing (wraps ParseErrorCode) 88 | /*! 89 | \ingroup RAPIDJSON_ERRORS 90 | \code 91 | Document doc; 92 | ParseResult ok = doc.Parse("[42]"); 93 | if (!ok) { 94 | fprintf(stderr, "JSON parse error: %s (%u)", 95 | GetParseError_En(ok.Code()), ok.Offset()); 96 | exit(EXIT_FAILURE); 97 | } 98 | \endcode 99 | \see GenericReader::Parse, GenericDocument::Parse 100 | */ 101 | struct ParseResult { 102 | 103 | //! Default constructor, no error. 104 | ParseResult() : code_(kParseErrorNone), offset_(0) {} 105 | //! Constructor to set an error. 106 | ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} 107 | 108 | //! Get the error code. 109 | ParseErrorCode Code() const { return code_; } 110 | //! Get the error offset, if \ref IsError(), 0 otherwise. 111 | size_t Offset() const { return offset_; } 112 | 113 | //! Conversion to \c bool, returns \c true, iff !\ref IsError(). 114 | operator bool() const { return !IsError(); } 115 | //! Whether the result is an error. 116 | bool IsError() const { return code_ != kParseErrorNone; } 117 | 118 | bool operator==(const ParseResult& that) const { return code_ == that.code_; } 119 | bool operator==(ParseErrorCode code) const { return code_ == code; } 120 | friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } 121 | 122 | //! Reset error code. 123 | void Clear() { Set(kParseErrorNone); } 124 | //! Update error code and offset. 125 | void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } 126 | 127 | private: 128 | ParseErrorCode code_; 129 | size_t offset_; 130 | }; 131 | 132 | //! Function pointer type of GetParseError(). 133 | /*! \ingroup RAPIDJSON_ERRORS 134 | 135 | This is the prototype for \c GetParseError_X(), where \c X is a locale. 136 | User can dynamically change locale in runtime, e.g.: 137 | \code 138 | GetParseErrorFunc GetParseError = GetParseError_En; // or whatever 139 | const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); 140 | \endcode 141 | */ 142 | typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); 143 | 144 | RAPIDJSON_NAMESPACE_END 145 | 146 | #endif // RAPIDJSON_ERROR_ERROR_H__ 147 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/filereadstream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEREADSTREAM_H_ 16 | #define RAPIDJSON_FILEREADSTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | #include 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! File byte stream for input using fread(). 24 | /*! 25 | \note implements Stream concept 26 | */ 27 | class FileReadStream { 28 | public: 29 | typedef char Ch; //!< Character type (byte). 30 | 31 | //! Constructor. 32 | /*! 33 | \param fp File pointer opened for read. 34 | \param buffer user-supplied buffer. 35 | \param bufferSize size of buffer in bytes. Must >=4 bytes. 36 | */ 37 | FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 38 | RAPIDJSON_ASSERT(fp_ != 0); 39 | RAPIDJSON_ASSERT(bufferSize >= 4); 40 | Read(); 41 | } 42 | 43 | Ch Peek() const { return *current_; } 44 | Ch Take() { Ch c = *current_; Read(); return c; } 45 | size_t Tell() const { return count_ + static_cast(current_ - buffer_); } 46 | 47 | // Not implemented 48 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 49 | void Flush() { RAPIDJSON_ASSERT(false); } 50 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 51 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 52 | 53 | // For encoding detection only. 54 | const Ch* Peek4() const { 55 | return (current_ + 4 <= bufferLast_) ? current_ : 0; 56 | } 57 | 58 | private: 59 | void Read() { 60 | if (current_ < bufferLast_) 61 | ++current_; 62 | else if (!eof_) { 63 | count_ += readCount_; 64 | readCount_ = fread(buffer_, 1, bufferSize_, fp_); 65 | bufferLast_ = buffer_ + readCount_ - 1; 66 | current_ = buffer_; 67 | 68 | if (readCount_ < bufferSize_) { 69 | buffer_[readCount_] = '\0'; 70 | ++bufferLast_; 71 | eof_ = true; 72 | } 73 | } 74 | } 75 | 76 | std::FILE* fp_; 77 | Ch *buffer_; 78 | size_t bufferSize_; 79 | Ch *bufferLast_; 80 | Ch *current_; 81 | size_t readCount_; 82 | size_t count_; //!< Number of characters read 83 | bool eof_; 84 | }; 85 | 86 | RAPIDJSON_NAMESPACE_END 87 | 88 | #endif // RAPIDJSON_FILESTREAM_H_ 89 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/filewritestream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEWRITESTREAM_H_ 16 | #define RAPIDJSON_FILEWRITESTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | #include 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! Wrapper of C file stream for input using fread(). 24 | /*! 25 | \note implements Stream concept 26 | */ 27 | class FileWriteStream { 28 | public: 29 | typedef char Ch; //!< Character type. Only support char. 30 | 31 | FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { 32 | RAPIDJSON_ASSERT(fp_ != 0); 33 | } 34 | 35 | void Put(char c) { 36 | if (current_ >= bufferEnd_) 37 | Flush(); 38 | 39 | *current_++ = c; 40 | } 41 | 42 | void PutN(char c, size_t n) { 43 | size_t avail = static_cast(bufferEnd_ - current_); 44 | while (n > avail) { 45 | std::memset(current_, c, avail); 46 | current_ += avail; 47 | Flush(); 48 | n -= avail; 49 | avail = static_cast(bufferEnd_ - current_); 50 | } 51 | 52 | if (n > 0) { 53 | std::memset(current_, c, n); 54 | current_ += n; 55 | } 56 | } 57 | 58 | void Flush() { 59 | if (current_ != buffer_) { 60 | size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); 61 | if (result < static_cast(current_ - buffer_)) { 62 | // failure deliberately ignored at this time 63 | // added to avoid warn_unused_result build errors 64 | } 65 | current_ = buffer_; 66 | } 67 | } 68 | 69 | // Not implemented 70 | char Peek() const { RAPIDJSON_ASSERT(false); return 0; } 71 | char Take() { RAPIDJSON_ASSERT(false); return 0; } 72 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 73 | char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 74 | size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } 75 | 76 | private: 77 | // Prohibit copy constructor & assignment operator. 78 | FileWriteStream(const FileWriteStream&); 79 | FileWriteStream& operator=(const FileWriteStream&); 80 | 81 | std::FILE* fp_; 82 | char *buffer_; 83 | char *bufferEnd_; 84 | char *current_; 85 | }; 86 | 87 | //! Implement specialized version of PutN() with memset() for better performance. 88 | template<> 89 | inline void PutN(FileWriteStream& stream, char c, size_t n) { 90 | stream.PutN(c, n); 91 | } 92 | 93 | RAPIDJSON_NAMESPACE_END 94 | 95 | #endif // RAPIDJSON_FILESTREAM_H_ 96 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/dtoa.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | // This is a C++ header-only implementation of Grisu2 algorithm from the publication: 16 | // Loitsch, Florian. "Printing floating-point numbers quickly and accurately with 17 | // integers." ACM Sigplan Notices 45.6 (2010): 233-243. 18 | 19 | #ifndef RAPIDJSON_DTOA_ 20 | #define RAPIDJSON_DTOA_ 21 | 22 | #include "itoa.h" // GetDigitsLut() 23 | #include "diyfp.h" 24 | #include "ieee754.h" 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | namespace internal { 28 | 29 | #ifdef __GNUC__ 30 | RAPIDJSON_DIAG_PUSH 31 | RAPIDJSON_DIAG_OFF(effc++) 32 | #endif 33 | 34 | inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { 35 | while (rest < wp_w && delta - rest >= ten_kappa && 36 | (rest + ten_kappa < wp_w || /// closer 37 | wp_w - rest > rest + ten_kappa - wp_w)) { 38 | buffer[len - 1]--; 39 | rest += ten_kappa; 40 | } 41 | } 42 | 43 | inline unsigned CountDecimalDigit32(uint32_t n) { 44 | // Simple pure C++ implementation was faster than __builtin_clz version in this situation. 45 | if (n < 10) return 1; 46 | if (n < 100) return 2; 47 | if (n < 1000) return 3; 48 | if (n < 10000) return 4; 49 | if (n < 100000) return 5; 50 | if (n < 1000000) return 6; 51 | if (n < 10000000) return 7; 52 | if (n < 100000000) return 8; 53 | // Will not reach 10 digits in DigitGen() 54 | //if (n < 1000000000) return 9; 55 | //return 10; 56 | return 9; 57 | } 58 | 59 | inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { 60 | static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; 61 | const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); 62 | const DiyFp wp_w = Mp - W; 63 | uint32_t p1 = static_cast(Mp.f >> -one.e); 64 | uint64_t p2 = Mp.f & (one.f - 1); 65 | unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] 66 | *len = 0; 67 | 68 | while (kappa > 0) { 69 | uint32_t d = 0; 70 | switch (kappa) { 71 | case 9: d = p1 / 100000000; p1 %= 100000000; break; 72 | case 8: d = p1 / 10000000; p1 %= 10000000; break; 73 | case 7: d = p1 / 1000000; p1 %= 1000000; break; 74 | case 6: d = p1 / 100000; p1 %= 100000; break; 75 | case 5: d = p1 / 10000; p1 %= 10000; break; 76 | case 4: d = p1 / 1000; p1 %= 1000; break; 77 | case 3: d = p1 / 100; p1 %= 100; break; 78 | case 2: d = p1 / 10; p1 %= 10; break; 79 | case 1: d = p1; p1 = 0; break; 80 | default:; 81 | } 82 | if (d || *len) 83 | buffer[(*len)++] = static_cast('0' + static_cast(d)); 84 | kappa--; 85 | uint64_t tmp = (static_cast(p1) << -one.e) + p2; 86 | if (tmp <= delta) { 87 | *K += kappa; 88 | GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); 89 | return; 90 | } 91 | } 92 | 93 | // kappa = 0 94 | for (;;) { 95 | p2 *= 10; 96 | delta *= 10; 97 | char d = static_cast(p2 >> -one.e); 98 | if (d || *len) 99 | buffer[(*len)++] = static_cast('0' + d); 100 | p2 &= one.f - 1; 101 | kappa--; 102 | if (p2 < delta) { 103 | *K += kappa; 104 | GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-static_cast(kappa)]); 105 | return; 106 | } 107 | } 108 | } 109 | 110 | inline void Grisu2(double value, char* buffer, int* length, int* K) { 111 | const DiyFp v(value); 112 | DiyFp w_m, w_p; 113 | v.NormalizedBoundaries(&w_m, &w_p); 114 | 115 | const DiyFp c_mk = GetCachedPower(w_p.e, K); 116 | const DiyFp W = v.Normalize() * c_mk; 117 | DiyFp Wp = w_p * c_mk; 118 | DiyFp Wm = w_m * c_mk; 119 | Wm.f++; 120 | Wp.f--; 121 | DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); 122 | } 123 | 124 | inline char* WriteExponent(int K, char* buffer) { 125 | if (K < 0) { 126 | *buffer++ = '-'; 127 | K = -K; 128 | } 129 | 130 | if (K >= 100) { 131 | *buffer++ = static_cast('0' + static_cast(K / 100)); 132 | K %= 100; 133 | const char* d = GetDigitsLut() + K * 2; 134 | *buffer++ = d[0]; 135 | *buffer++ = d[1]; 136 | } 137 | else if (K >= 10) { 138 | const char* d = GetDigitsLut() + K * 2; 139 | *buffer++ = d[0]; 140 | *buffer++ = d[1]; 141 | } 142 | else 143 | *buffer++ = static_cast('0' + static_cast(K)); 144 | 145 | return buffer; 146 | } 147 | 148 | inline char* Prettify(char* buffer, int length, int k) { 149 | const int kk = length + k; // 10^(kk-1) <= v < 10^kk 150 | 151 | if (length <= kk && kk <= 21) { 152 | // 1234e7 -> 12340000000 153 | for (int i = length; i < kk; i++) 154 | buffer[i] = '0'; 155 | buffer[kk] = '.'; 156 | buffer[kk + 1] = '0'; 157 | return &buffer[kk + 2]; 158 | } 159 | else if (0 < kk && kk <= 21) { 160 | // 1234e-2 -> 12.34 161 | std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); 162 | buffer[kk] = '.'; 163 | return &buffer[length + 1]; 164 | } 165 | else if (-6 < kk && kk <= 0) { 166 | // 1234e-6 -> 0.001234 167 | const int offset = 2 - kk; 168 | std::memmove(&buffer[offset], &buffer[0], static_cast(length)); 169 | buffer[0] = '0'; 170 | buffer[1] = '.'; 171 | for (int i = 2; i < offset; i++) 172 | buffer[i] = '0'; 173 | return &buffer[length + offset]; 174 | } 175 | else if (length == 1) { 176 | // 1e30 177 | buffer[1] = 'e'; 178 | return WriteExponent(kk - 1, &buffer[2]); 179 | } 180 | else { 181 | // 1234e30 -> 1.234e33 182 | std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); 183 | buffer[1] = '.'; 184 | buffer[length + 1] = 'e'; 185 | return WriteExponent(kk - 1, &buffer[0 + length + 2]); 186 | } 187 | } 188 | 189 | inline char* dtoa(double value, char* buffer) { 190 | Double d(value); 191 | if (d.IsZero()) { 192 | if (d.Sign()) 193 | *buffer++ = '-'; // -0.0, Issue #289 194 | buffer[0] = '0'; 195 | buffer[1] = '.'; 196 | buffer[2] = '0'; 197 | return &buffer[3]; 198 | } 199 | else { 200 | if (value < 0) { 201 | *buffer++ = '-'; 202 | value = -value; 203 | } 204 | int length, K; 205 | Grisu2(value, buffer, &length, &K); 206 | return Prettify(buffer, length, K); 207 | } 208 | } 209 | 210 | #ifdef __GNUC__ 211 | RAPIDJSON_DIAG_POP 212 | #endif 213 | 214 | } // namespace internal 215 | RAPIDJSON_NAMESPACE_END 216 | 217 | #endif // RAPIDJSON_DTOA_ 218 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/ieee754.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_IEEE754_ 16 | #define RAPIDJSON_IEEE754_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | class Double { 24 | public: 25 | Double() {} 26 | Double(double d) : d_(d) {} 27 | Double(uint64_t u) : u_(u) {} 28 | 29 | double Value() const { return d_; } 30 | uint64_t Uint64Value() const { return u_; } 31 | 32 | double NextPositiveDouble() const { 33 | RAPIDJSON_ASSERT(!Sign()); 34 | return Double(u_ + 1).Value(); 35 | } 36 | 37 | bool Sign() const { return (u_ & kSignMask) != 0; } 38 | uint64_t Significand() const { return u_ & kSignificandMask; } 39 | int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } 40 | 41 | bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } 42 | bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } 43 | bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } 44 | bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } 45 | 46 | uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } 47 | int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } 48 | uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } 49 | 50 | static unsigned EffectiveSignificandSize(int order) { 51 | if (order >= -1021) 52 | return 53; 53 | else if (order <= -1074) 54 | return 0; 55 | else 56 | return (unsigned)order + 1074; 57 | } 58 | 59 | private: 60 | static const int kSignificandSize = 52; 61 | static const int kExponentBias = 0x3FF; 62 | static const int kDenormalExponent = 1 - kExponentBias; 63 | static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); 64 | static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); 65 | static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); 66 | static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); 67 | 68 | union { 69 | double d_; 70 | uint64_t u_; 71 | }; 72 | }; 73 | 74 | } // namespace internal 75 | RAPIDJSON_NAMESPACE_END 76 | 77 | #endif // RAPIDJSON_IEEE754_ 78 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/meta.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_META_H_ 16 | #define RAPIDJSON_INTERNAL_META_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | #ifdef __GNUC__ 21 | RAPIDJSON_DIAG_PUSH 22 | RAPIDJSON_DIAG_OFF(effc++) 23 | #endif 24 | #if defined(_MSC_VER) 25 | RAPIDJSON_DIAG_PUSH 26 | RAPIDJSON_DIAG_OFF(6334) 27 | #endif 28 | 29 | #if RAPIDJSON_HAS_CXX11_TYPETRAITS 30 | #include 31 | #endif 32 | 33 | //@cond RAPIDJSON_INTERNAL 34 | RAPIDJSON_NAMESPACE_BEGIN 35 | namespace internal { 36 | 37 | // Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching 38 | template struct Void { typedef void Type; }; 39 | 40 | /////////////////////////////////////////////////////////////////////////////// 41 | // BoolType, TrueType, FalseType 42 | // 43 | template struct BoolType { 44 | static const bool Value = Cond; 45 | typedef BoolType Type; 46 | }; 47 | typedef BoolType TrueType; 48 | typedef BoolType FalseType; 49 | 50 | 51 | /////////////////////////////////////////////////////////////////////////////// 52 | // SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr 53 | // 54 | 55 | template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; 56 | template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; 57 | template struct SelectIfCond : SelectIfImpl::template Apply {}; 58 | template struct SelectIf : SelectIfCond {}; 59 | 60 | template struct AndExprCond : FalseType {}; 61 | template <> struct AndExprCond : TrueType {}; 62 | template struct OrExprCond : TrueType {}; 63 | template <> struct OrExprCond : FalseType {}; 64 | 65 | template struct BoolExpr : SelectIf::Type {}; 66 | template struct NotExpr : SelectIf::Type {}; 67 | template struct AndExpr : AndExprCond::Type {}; 68 | template struct OrExpr : OrExprCond::Type {}; 69 | 70 | 71 | /////////////////////////////////////////////////////////////////////////////// 72 | // AddConst, MaybeAddConst, RemoveConst 73 | template struct AddConst { typedef const T Type; }; 74 | template struct MaybeAddConst : SelectIfCond {}; 75 | template struct RemoveConst { typedef T Type; }; 76 | template struct RemoveConst { typedef T Type; }; 77 | 78 | 79 | /////////////////////////////////////////////////////////////////////////////// 80 | // IsSame, IsConst, IsMoreConst, IsPointer 81 | // 82 | template struct IsSame : FalseType {}; 83 | template struct IsSame : TrueType {}; 84 | 85 | template struct IsConst : FalseType {}; 86 | template struct IsConst : TrueType {}; 87 | 88 | template 89 | struct IsMoreConst 90 | : AndExpr::Type, typename RemoveConst::Type>, 91 | BoolType::Value >= IsConst::Value> >::Type {}; 92 | 93 | template struct IsPointer : FalseType {}; 94 | template struct IsPointer : TrueType {}; 95 | 96 | /////////////////////////////////////////////////////////////////////////////// 97 | // IsBaseOf 98 | // 99 | #if RAPIDJSON_HAS_CXX11_TYPETRAITS 100 | 101 | template struct IsBaseOf 102 | : BoolType< ::std::is_base_of::value> {}; 103 | 104 | #else // simplified version adopted from Boost 105 | 106 | template struct IsBaseOfImpl { 107 | RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); 108 | RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); 109 | 110 | typedef char (&Yes)[1]; 111 | typedef char (&No) [2]; 112 | 113 | template 114 | static Yes Check(const D*, T); 115 | static No Check(const B*, int); 116 | 117 | struct Host { 118 | operator const B*() const; 119 | operator const D*(); 120 | }; 121 | 122 | enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; 123 | }; 124 | 125 | template struct IsBaseOf 126 | : OrExpr, BoolExpr > >::Type {}; 127 | 128 | #endif // RAPIDJSON_HAS_CXX11_TYPETRAITS 129 | 130 | 131 | ////////////////////////////////////////////////////////////////////////// 132 | // EnableIf / DisableIf 133 | // 134 | template struct EnableIfCond { typedef T Type; }; 135 | template struct EnableIfCond { /* empty */ }; 136 | 137 | template struct DisableIfCond { typedef T Type; }; 138 | template struct DisableIfCond { /* empty */ }; 139 | 140 | template 141 | struct EnableIf : EnableIfCond {}; 142 | 143 | template 144 | struct DisableIf : DisableIfCond {}; 145 | 146 | // SFINAE helpers 147 | struct SfinaeTag {}; 148 | template struct RemoveSfinaeTag; 149 | template struct RemoveSfinaeTag { typedef T Type; }; 150 | 151 | #define RAPIDJSON_REMOVEFPTR_(type) \ 152 | typename ::RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ 153 | < ::RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type 154 | 155 | #define RAPIDJSON_ENABLEIF(cond) \ 156 | typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ 157 | ::Type * = NULL 158 | 159 | #define RAPIDJSON_DISABLEIF(cond) \ 160 | typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ 161 | ::Type * = NULL 162 | 163 | #define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ 164 | typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ 165 | ::Type 167 | 168 | #define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ 169 | typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ 170 | ::Type 172 | 173 | } // namespace internal 174 | RAPIDJSON_NAMESPACE_END 175 | //@endcond 176 | 177 | #if defined(__GNUC__) || defined(_MSC_VER) 178 | RAPIDJSON_DIAG_POP 179 | #endif 180 | 181 | #endif // RAPIDJSON_INTERNAL_META_H_ 182 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/pow10.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_POW10_ 16 | #define RAPIDJSON_POW10_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Computes integer powers of 10 in double (10.0^n). 24 | /*! This function uses lookup table for fast and accurate results. 25 | \param n non-negative exponent. Must <= 308. 26 | \return 10.0^n 27 | */ 28 | inline double Pow10(int n) { 29 | static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes 30 | 1e+0, 31 | 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 32 | 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 33 | 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 34 | 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 35 | 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 36 | 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, 37 | 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, 38 | 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, 39 | 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, 40 | 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, 41 | 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, 42 | 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, 43 | 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, 44 | 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, 45 | 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, 46 | 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 47 | }; 48 | RAPIDJSON_ASSERT(n >= 0 && n <= 308); 49 | return e[n]; 50 | } 51 | 52 | } // namespace internal 53 | RAPIDJSON_NAMESPACE_END 54 | 55 | #endif // RAPIDJSON_POW10_ 56 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/stack.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_STACK_H_ 16 | #define RAPIDJSON_INTERNAL_STACK_H_ 17 | 18 | #include "../rapidjson.h" 19 | #include "swap.h" 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | namespace internal { 23 | 24 | /////////////////////////////////////////////////////////////////////////////// 25 | // Stack 26 | 27 | //! A type-unsafe stack for storing different types of data. 28 | /*! \tparam Allocator Allocator for allocating stack memory. 29 | */ 30 | template 31 | class Stack { 32 | public: 33 | // Optimization note: Do not allocate memory for stack_ in constructor. 34 | // Do it lazily when first Push() -> Expand() -> Resize(). 35 | Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { 36 | RAPIDJSON_ASSERT(stackCapacity > 0); 37 | } 38 | 39 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 40 | Stack(Stack&& rhs) 41 | : allocator_(rhs.allocator_), 42 | ownAllocator_(rhs.ownAllocator_), 43 | stack_(rhs.stack_), 44 | stackTop_(rhs.stackTop_), 45 | stackEnd_(rhs.stackEnd_), 46 | initialCapacity_(rhs.initialCapacity_) 47 | { 48 | rhs.allocator_ = 0; 49 | rhs.ownAllocator_ = 0; 50 | rhs.stack_ = 0; 51 | rhs.stackTop_ = 0; 52 | rhs.stackEnd_ = 0; 53 | rhs.initialCapacity_ = 0; 54 | } 55 | #endif 56 | 57 | ~Stack() { 58 | Destroy(); 59 | } 60 | 61 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 62 | Stack& operator=(Stack&& rhs) { 63 | if (&rhs != this) 64 | { 65 | Destroy(); 66 | 67 | allocator_ = rhs.allocator_; 68 | ownAllocator_ = rhs.ownAllocator_; 69 | stack_ = rhs.stack_; 70 | stackTop_ = rhs.stackTop_; 71 | stackEnd_ = rhs.stackEnd_; 72 | initialCapacity_ = rhs.initialCapacity_; 73 | 74 | rhs.allocator_ = 0; 75 | rhs.ownAllocator_ = 0; 76 | rhs.stack_ = 0; 77 | rhs.stackTop_ = 0; 78 | rhs.stackEnd_ = 0; 79 | rhs.initialCapacity_ = 0; 80 | } 81 | return *this; 82 | } 83 | #endif 84 | 85 | void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { 86 | internal::Swap(allocator_, rhs.allocator_); 87 | internal::Swap(ownAllocator_, rhs.ownAllocator_); 88 | internal::Swap(stack_, rhs.stack_); 89 | internal::Swap(stackTop_, rhs.stackTop_); 90 | internal::Swap(stackEnd_, rhs.stackEnd_); 91 | internal::Swap(initialCapacity_, rhs.initialCapacity_); 92 | } 93 | 94 | void Clear() { stackTop_ = stack_; } 95 | 96 | void ShrinkToFit() { 97 | if (Empty()) { 98 | // If the stack is empty, completely deallocate the memory. 99 | Allocator::Free(stack_); 100 | stack_ = 0; 101 | stackTop_ = 0; 102 | stackEnd_ = 0; 103 | } 104 | else 105 | Resize(GetSize()); 106 | } 107 | 108 | // Optimization note: try to minimize the size of this function for force inline. 109 | // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. 110 | template 111 | RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { 112 | // Expand the stack if needed 113 | if (stackTop_ + sizeof(T) * count >= stackEnd_) 114 | Expand(count); 115 | 116 | T* ret = reinterpret_cast(stackTop_); 117 | stackTop_ += sizeof(T) * count; 118 | return ret; 119 | } 120 | 121 | template 122 | T* Pop(size_t count) { 123 | RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); 124 | stackTop_ -= count * sizeof(T); 125 | return reinterpret_cast(stackTop_); 126 | } 127 | 128 | template 129 | T* Top() { 130 | RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); 131 | return reinterpret_cast(stackTop_ - sizeof(T)); 132 | } 133 | 134 | template 135 | T* Bottom() { return (T*)stack_; } 136 | 137 | bool HasAllocator() const { 138 | return allocator_ != 0; 139 | } 140 | 141 | Allocator& GetAllocator() { 142 | RAPIDJSON_ASSERT(allocator_); 143 | return *allocator_; 144 | } 145 | bool Empty() const { return stackTop_ == stack_; } 146 | size_t GetSize() const { return static_cast(stackTop_ - stack_); } 147 | size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } 148 | 149 | private: 150 | template 151 | void Expand(size_t count) { 152 | // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. 153 | size_t newCapacity; 154 | if (stack_ == 0) { 155 | if (!allocator_) 156 | ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); 157 | newCapacity = initialCapacity_; 158 | } else { 159 | newCapacity = GetCapacity(); 160 | newCapacity += (newCapacity + 1) / 2; 161 | } 162 | size_t newSize = GetSize() + sizeof(T) * count; 163 | if (newCapacity < newSize) 164 | newCapacity = newSize; 165 | 166 | Resize(newCapacity); 167 | } 168 | 169 | void Resize(size_t newCapacity) { 170 | const size_t size = GetSize(); // Backup the current size 171 | stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); 172 | stackTop_ = stack_ + size; 173 | stackEnd_ = stack_ + newCapacity; 174 | } 175 | 176 | void Destroy() { 177 | Allocator::Free(stack_); 178 | RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack 179 | } 180 | 181 | // Prohibit copy constructor & assignment operator. 182 | Stack(const Stack&); 183 | Stack& operator=(const Stack&); 184 | 185 | Allocator* allocator_; 186 | Allocator* ownAllocator_; 187 | char *stack_; 188 | char *stackTop_; 189 | char *stackEnd_; 190 | size_t initialCapacity_; 191 | }; 192 | 193 | } // namespace internal 194 | RAPIDJSON_NAMESPACE_END 195 | 196 | #endif // RAPIDJSON_STACK_H_ 197 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/strfunc.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ 16 | #define RAPIDJSON_INTERNAL_STRFUNC_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Custom strlen() which works on different character types. 24 | /*! \tparam Ch Character type (e.g. char, wchar_t, short) 25 | \param s Null-terminated input string. 26 | \return Number of characters in the string. 27 | \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. 28 | */ 29 | template 30 | inline SizeType StrLen(const Ch* s) { 31 | const Ch* p = s; 32 | while (*p) ++p; 33 | return SizeType(p - s); 34 | } 35 | 36 | } // namespace internal 37 | RAPIDJSON_NAMESPACE_END 38 | 39 | #endif // RAPIDJSON_INTERNAL_STRFUNC_H_ 40 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/strtod.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_STRTOD_ 16 | #define RAPIDJSON_STRTOD_ 17 | 18 | #include "../rapidjson.h" 19 | #include "ieee754.h" 20 | #include "biginteger.h" 21 | #include "diyfp.h" 22 | #include "pow10.h" 23 | 24 | RAPIDJSON_NAMESPACE_BEGIN 25 | namespace internal { 26 | 27 | inline double FastPath(double significand, int exp) { 28 | if (exp < -308) 29 | return 0.0; 30 | else if (exp >= 0) 31 | return significand * internal::Pow10(exp); 32 | else 33 | return significand / internal::Pow10(-exp); 34 | } 35 | 36 | inline double StrtodNormalPrecision(double d, int p) { 37 | if (p < -308) { 38 | // Prevent expSum < -308, making Pow10(p) = 0 39 | d = FastPath(d, -308); 40 | d = FastPath(d, p + 308); 41 | } 42 | else 43 | d = FastPath(d, p); 44 | return d; 45 | } 46 | 47 | template 48 | inline T Min3(T a, T b, T c) { 49 | T m = a; 50 | if (m > b) m = b; 51 | if (m > c) m = c; 52 | return m; 53 | } 54 | 55 | inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { 56 | const Double db(b); 57 | const uint64_t bInt = db.IntegerSignificand(); 58 | const int bExp = db.IntegerExponent(); 59 | const int hExp = bExp - 1; 60 | 61 | int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; 62 | 63 | // Adjust for decimal exponent 64 | if (dExp >= 0) { 65 | dS_Exp2 += dExp; 66 | dS_Exp5 += dExp; 67 | } 68 | else { 69 | bS_Exp2 -= dExp; 70 | bS_Exp5 -= dExp; 71 | hS_Exp2 -= dExp; 72 | hS_Exp5 -= dExp; 73 | } 74 | 75 | // Adjust for binary exponent 76 | if (bExp >= 0) 77 | bS_Exp2 += bExp; 78 | else { 79 | dS_Exp2 -= bExp; 80 | hS_Exp2 -= bExp; 81 | } 82 | 83 | // Adjust for half ulp exponent 84 | if (hExp >= 0) 85 | hS_Exp2 += hExp; 86 | else { 87 | dS_Exp2 -= hExp; 88 | bS_Exp2 -= hExp; 89 | } 90 | 91 | // Remove common power of two factor from all three scaled values 92 | int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); 93 | dS_Exp2 -= common_Exp2; 94 | bS_Exp2 -= common_Exp2; 95 | hS_Exp2 -= common_Exp2; 96 | 97 | BigInteger dS = d; 98 | dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); 99 | 100 | BigInteger bS(bInt); 101 | bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); 102 | 103 | BigInteger hS(1); 104 | hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); 105 | 106 | BigInteger delta(0); 107 | dS.Difference(bS, &delta); 108 | 109 | return delta.Compare(hS); 110 | } 111 | 112 | inline bool StrtodFast(double d, int p, double* result) { 113 | // Use fast path for string-to-double conversion if possible 114 | // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ 115 | if (p > 22 && p < 22 + 16) { 116 | // Fast Path Cases In Disguise 117 | d *= internal::Pow10(p - 22); 118 | p = 22; 119 | } 120 | 121 | if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 122 | *result = FastPath(d, p); 123 | return true; 124 | } 125 | else 126 | return false; 127 | } 128 | 129 | // Compute an approximation and see if it is within 1/2 ULP 130 | inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { 131 | uint64_t significand = 0; 132 | size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 133 | for (; i < length; i++) { 134 | if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || 135 | (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) 136 | break; 137 | significand = significand * 10u + static_cast(decimals[i] - '0'); 138 | } 139 | 140 | if (i < length && decimals[i] >= '5') // Rounding 141 | significand++; 142 | 143 | size_t remaining = length - i; 144 | const unsigned kUlpShift = 3; 145 | const unsigned kUlp = 1 << kUlpShift; 146 | int error = (remaining == 0) ? 0 : kUlp / 2; 147 | 148 | DiyFp v(significand, 0); 149 | v = v.Normalize(); 150 | error <<= -v.e; 151 | 152 | const int dExp = (int)decimalPosition - (int)i + exp; 153 | 154 | int actualExp; 155 | DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); 156 | if (actualExp != dExp) { 157 | static const DiyFp kPow10[] = { 158 | DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 159 | DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 160 | DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 161 | DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 162 | DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 163 | DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 164 | DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 165 | }; 166 | int adjustment = dExp - actualExp - 1; 167 | RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); 168 | v = v * kPow10[adjustment]; 169 | if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit 170 | error += kUlp / 2; 171 | } 172 | 173 | v = v * cachedPower; 174 | 175 | error += kUlp + (error == 0 ? 0 : 1); 176 | 177 | const int oldExp = v.e; 178 | v = v.Normalize(); 179 | error <<= oldExp - v.e; 180 | 181 | const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); 182 | unsigned precisionSize = 64 - effectiveSignificandSize; 183 | if (precisionSize + kUlpShift >= 64) { 184 | unsigned scaleExp = (precisionSize + kUlpShift) - 63; 185 | v.f >>= scaleExp; 186 | v.e += scaleExp; 187 | error = (error >> scaleExp) + 1 + static_cast(kUlp); 188 | precisionSize -= scaleExp; 189 | } 190 | 191 | DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); 192 | const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; 193 | const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; 194 | if (precisionBits >= halfWay + static_cast(error)) { 195 | rounded.f++; 196 | if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) 197 | rounded.f >>= 1; 198 | rounded.e++; 199 | } 200 | } 201 | 202 | *result = rounded.ToDouble(); 203 | 204 | return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); 205 | } 206 | 207 | inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { 208 | const BigInteger dInt(decimals, length); 209 | const int dExp = (int)decimalPosition - (int)length + exp; 210 | Double a(approx); 211 | int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); 212 | if (cmp < 0) 213 | return a.Value(); // within half ULP 214 | else if (cmp == 0) { 215 | // Round towards even 216 | if (a.Significand() & 1) 217 | return a.NextPositiveDouble(); 218 | else 219 | return a.Value(); 220 | } 221 | else // adjustment 222 | return a.NextPositiveDouble(); 223 | } 224 | 225 | inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { 226 | RAPIDJSON_ASSERT(d >= 0.0); 227 | RAPIDJSON_ASSERT(length >= 1); 228 | 229 | double result; 230 | if (StrtodFast(d, p, &result)) 231 | return result; 232 | 233 | // Trim leading zeros 234 | while (*decimals == '0' && length > 1) { 235 | length--; 236 | decimals++; 237 | decimalPosition--; 238 | } 239 | 240 | // Trim trailing zeros 241 | while (decimals[length - 1] == '0' && length > 1) { 242 | length--; 243 | decimalPosition--; 244 | exp++; 245 | } 246 | 247 | // Trim right-most digits 248 | const int kMaxDecimalDigit = 780; 249 | if ((int)length > kMaxDecimalDigit) { 250 | int delta = (int(length) - kMaxDecimalDigit); 251 | exp += delta; 252 | decimalPosition -= static_cast(delta); 253 | length = kMaxDecimalDigit; 254 | } 255 | 256 | // If too small, underflow to zero 257 | if (int(length) + exp < -324) 258 | return 0.0; 259 | 260 | if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) 261 | return result; 262 | 263 | // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison 264 | return StrtodBigInteger(result, decimals, length, decimalPosition, exp); 265 | } 266 | 267 | } // namespace internal 268 | RAPIDJSON_NAMESPACE_END 269 | 270 | #endif // RAPIDJSON_STRTOD_ 271 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/internal/swap.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_SWAP_H_ 16 | #define RAPIDJSON_INTERNAL_SWAP_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Custom swap() to avoid dependency on C++ header 24 | /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. 25 | \note This has the same semantics as std::swap(). 26 | */ 27 | template 28 | inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { 29 | T tmp = a; 30 | a = b; 31 | b = tmp; 32 | } 33 | 34 | } // namespace internal 35 | RAPIDJSON_NAMESPACE_END 36 | 37 | #endif // RAPIDJSON_INTERNAL_SWAP_H_ 38 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/memorybuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYBUFFER_H_ 16 | #define RAPIDJSON_MEMORYBUFFER_H_ 17 | 18 | #include "rapidjson.h" 19 | #include "internal/stack.h" 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! Represents an in-memory output byte stream. 24 | /*! 25 | This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. 26 | 27 | It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. 28 | 29 | Differences between MemoryBuffer and StringBuffer: 30 | 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. 31 | 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. 32 | 33 | \tparam Allocator type for allocating memory buffer. 34 | \note implements Stream concept 35 | */ 36 | template 37 | struct GenericMemoryBuffer { 38 | typedef char Ch; // byte 39 | 40 | GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 41 | 42 | void Put(Ch c) { *stack_.template Push() = c; } 43 | void Flush() {} 44 | 45 | void Clear() { stack_.Clear(); } 46 | void ShrinkToFit() { stack_.ShrinkToFit(); } 47 | Ch* Push(size_t count) { return stack_.template Push(count); } 48 | void Pop(size_t count) { stack_.template Pop(count); } 49 | 50 | const Ch* GetBuffer() const { 51 | return stack_.template Bottom(); 52 | } 53 | 54 | size_t GetSize() const { return stack_.GetSize(); } 55 | 56 | static const size_t kDefaultCapacity = 256; 57 | mutable internal::Stack stack_; 58 | }; 59 | 60 | typedef GenericMemoryBuffer<> MemoryBuffer; 61 | 62 | //! Implement specialized version of PutN() with memset() for better performance. 63 | template<> 64 | inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { 65 | std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); 66 | } 67 | 68 | RAPIDJSON_NAMESPACE_END 69 | 70 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 71 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/memorystream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYSTREAM_H_ 16 | #define RAPIDJSON_MEMORYSTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | 22 | //! Represents an in-memory input byte stream. 23 | /*! 24 | This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. 25 | 26 | It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. 27 | 28 | Differences between MemoryStream and StringStream: 29 | 1. StringStream has encoding but MemoryStream is a byte stream. 30 | 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. 31 | 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). 32 | \note implements Stream concept 33 | */ 34 | struct MemoryStream { 35 | typedef char Ch; // byte 36 | 37 | MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} 38 | 39 | Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } 40 | Ch Take() { return (src_ == end_) ? '\0' : *src_++; } 41 | size_t Tell() const { return static_cast(src_ - begin_); } 42 | 43 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 44 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 45 | void Flush() { RAPIDJSON_ASSERT(false); } 46 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 47 | 48 | // For encoding detection only. 49 | const Ch* Peek4() const { 50 | return Tell() + 4 <= size_ ? src_ : 0; 51 | } 52 | 53 | const Ch* src_; //!< Current read position. 54 | const Ch* begin_; //!< Original head of the string. 55 | const Ch* end_; //!< End of stream. 56 | size_t size_; //!< Size of the stream. 57 | }; 58 | 59 | RAPIDJSON_NAMESPACE_END 60 | 61 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 62 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/prettywriter.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_PRETTYWRITER_H_ 16 | #define RAPIDJSON_PRETTYWRITER_H_ 17 | 18 | #include "writer.h" 19 | 20 | #ifdef __GNUC__ 21 | RAPIDJSON_DIAG_PUSH 22 | RAPIDJSON_DIAG_OFF(effc++) 23 | #endif 24 | 25 | RAPIDJSON_NAMESPACE_BEGIN 26 | 27 | //! Writer with indentation and spacing. 28 | /*! 29 | \tparam OutputStream Type of ouptut os. 30 | \tparam SourceEncoding Encoding of source string. 31 | \tparam TargetEncoding Encoding of output stream. 32 | \tparam StackAllocator Type of allocator for allocating memory of stack. 33 | */ 34 | template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> 35 | class PrettyWriter : public Writer { 36 | public: 37 | typedef Writer Base; 38 | typedef typename Base::Ch Ch; 39 | 40 | //! Constructor 41 | /*! \param os Output stream. 42 | \param allocator User supplied allocator. If it is null, it will create a private one. 43 | \param levelDepth Initial capacity of stack. 44 | */ 45 | PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : 46 | Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} 47 | 48 | //! Set custom indentation. 49 | /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). 50 | \param indentCharCount Number of indent characters for each indentation level. 51 | \note The default indentation is 4 spaces. 52 | */ 53 | PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { 54 | RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); 55 | indentChar_ = indentChar; 56 | indentCharCount_ = indentCharCount; 57 | return *this; 58 | } 59 | 60 | /*! @name Implementation of Handler 61 | \see Handler 62 | */ 63 | //@{ 64 | 65 | bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } 66 | bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } 67 | bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } 68 | bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } 69 | bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } 70 | bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } 71 | bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } 72 | 73 | bool String(const Ch* str, SizeType length, bool copy = false) { 74 | (void)copy; 75 | PrettyPrefix(kStringType); 76 | return Base::WriteString(str, length); 77 | } 78 | 79 | #if RAPIDJSON_HAS_STDSTRING 80 | bool String(const std::basic_string& str) { 81 | return String(str.data(), SizeType(str.size())); 82 | } 83 | #endif 84 | 85 | bool StartObject() { 86 | PrettyPrefix(kObjectType); 87 | new (Base::level_stack_.template Push()) typename Base::Level(false); 88 | return Base::WriteStartObject(); 89 | } 90 | 91 | bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } 92 | 93 | bool EndObject(SizeType memberCount = 0) { 94 | (void)memberCount; 95 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); 96 | RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); 97 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 98 | 99 | if (!empty) { 100 | Base::os_->Put('\n'); 101 | WriteIndent(); 102 | } 103 | bool ret = Base::WriteEndObject(); 104 | (void)ret; 105 | RAPIDJSON_ASSERT(ret == true); 106 | if (Base::level_stack_.Empty()) // end of json text 107 | Base::os_->Flush(); 108 | return true; 109 | } 110 | 111 | bool StartArray() { 112 | PrettyPrefix(kArrayType); 113 | new (Base::level_stack_.template Push()) typename Base::Level(true); 114 | return Base::WriteStartArray(); 115 | } 116 | 117 | bool EndArray(SizeType memberCount = 0) { 118 | (void)memberCount; 119 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); 120 | RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); 121 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 122 | 123 | if (!empty) { 124 | Base::os_->Put('\n'); 125 | WriteIndent(); 126 | } 127 | bool ret = Base::WriteEndArray(); 128 | (void)ret; 129 | RAPIDJSON_ASSERT(ret == true); 130 | if (Base::level_stack_.Empty()) // end of json text 131 | Base::os_->Flush(); 132 | return true; 133 | } 134 | 135 | //@} 136 | 137 | /*! @name Convenience extensions */ 138 | //@{ 139 | 140 | //! Simpler but slower overload. 141 | bool String(const Ch* str) { return String(str, internal::StrLen(str)); } 142 | bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } 143 | 144 | //@} 145 | protected: 146 | void PrettyPrefix(Type type) { 147 | (void)type; 148 | if (Base::level_stack_.GetSize() != 0) { // this value is not at root 149 | typename Base::Level* level = Base::level_stack_.template Top(); 150 | 151 | if (level->inArray) { 152 | if (level->valueCount > 0) { 153 | Base::os_->Put(','); // add comma if it is not the first element in array 154 | Base::os_->Put('\n'); 155 | } 156 | else 157 | Base::os_->Put('\n'); 158 | WriteIndent(); 159 | } 160 | else { // in object 161 | if (level->valueCount > 0) { 162 | if (level->valueCount % 2 == 0) { 163 | Base::os_->Put(','); 164 | Base::os_->Put('\n'); 165 | } 166 | else { 167 | Base::os_->Put(':'); 168 | Base::os_->Put(' '); 169 | } 170 | } 171 | else 172 | Base::os_->Put('\n'); 173 | 174 | if (level->valueCount % 2 == 0) 175 | WriteIndent(); 176 | } 177 | if (!level->inArray && level->valueCount % 2 == 0) 178 | RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name 179 | level->valueCount++; 180 | } 181 | else { 182 | RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. 183 | Base::hasRoot_ = true; 184 | } 185 | } 186 | 187 | void WriteIndent() { 188 | size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; 189 | PutN(*Base::os_, indentChar_, count); 190 | } 191 | 192 | Ch indentChar_; 193 | unsigned indentCharCount_; 194 | 195 | private: 196 | // Prohibit copy constructor & assignment operator. 197 | PrettyWriter(const PrettyWriter&); 198 | PrettyWriter& operator=(const PrettyWriter&); 199 | }; 200 | 201 | RAPIDJSON_NAMESPACE_END 202 | 203 | #ifdef __GNUC__ 204 | RAPIDJSON_DIAG_POP 205 | #endif 206 | 207 | #endif // RAPIDJSON_RAPIDJSON_H_ 208 | -------------------------------------------------------------------------------- /Valkyrie/include/rapidjson/stringbuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_STRINGBUFFER_H_ 16 | #define RAPIDJSON_STRINGBUFFER_H_ 17 | 18 | #include "rapidjson.h" 19 | 20 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 21 | #include // std::move 22 | #endif 23 | 24 | #include "internal/stack.h" 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | 28 | //! Represents an in-memory output stream. 29 | /*! 30 | \tparam Encoding Encoding of the stream. 31 | \tparam Allocator type for allocating memory buffer. 32 | \note implements Stream concept 33 | */ 34 | template 35 | class GenericStringBuffer { 36 | public: 37 | typedef typename Encoding::Ch Ch; 38 | 39 | GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 40 | 41 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 42 | GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} 43 | GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { 44 | if (&rhs != this) 45 | stack_ = std::move(rhs.stack_); 46 | return *this; 47 | } 48 | #endif 49 | 50 | void Put(Ch c) { *stack_.template Push() = c; } 51 | void Flush() {} 52 | 53 | void Clear() { stack_.Clear(); } 54 | void ShrinkToFit() { 55 | // Push and pop a null terminator. This is safe. 56 | *stack_.template Push() = '\0'; 57 | stack_.ShrinkToFit(); 58 | stack_.template Pop(1); 59 | } 60 | Ch* Push(size_t count) { return stack_.template Push(count); } 61 | void Pop(size_t count) { stack_.template Pop(count); } 62 | 63 | const Ch* GetString() const { 64 | // Push and pop a null terminator. This is safe. 65 | *stack_.template Push() = '\0'; 66 | stack_.template Pop(1); 67 | 68 | return stack_.template Bottom(); 69 | } 70 | 71 | size_t GetSize() const { return stack_.GetSize(); } 72 | 73 | static const size_t kDefaultCapacity = 256; 74 | mutable internal::Stack stack_; 75 | 76 | private: 77 | // Prohibit copy constructor & assignment operator. 78 | GenericStringBuffer(const GenericStringBuffer&); 79 | GenericStringBuffer& operator=(const GenericStringBuffer&); 80 | }; 81 | 82 | //! String buffer with UTF8 encoding 83 | typedef GenericStringBuffer > StringBuffer; 84 | 85 | //! Implement specialized version of PutN() with memset() for better performance. 86 | template<> 87 | inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { 88 | std::memset(stream.stack_.Push(n), c, n * sizeof(c)); 89 | } 90 | 91 | RAPIDJSON_NAMESPACE_END 92 | 93 | #endif // RAPIDJSON_STRINGBUFFER_H_ 94 | -------------------------------------------------------------------------------- /Valkyrie/src/ExpressionParser.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/ExpressionParser.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace valkyrie; 9 | 10 | static void raise_exception() { 11 | cout << "Parse Exception!" << endl; 12 | exit(-1); 13 | } 14 | 15 | ExpressionParser::ExpressionParser(){ 16 | arithmeticOperators.insert("+"); 17 | arithmeticOperators.insert("-"); 18 | arithmeticOperators.insert("*"); 19 | arithmeticOperators.insert("/"); 20 | relationalOperators.insert("="); 21 | relationalOperators.insert("<>"); 22 | relationalOperators.insert(">"); 23 | relationalOperators.insert(">="); 24 | relationalOperators.insert("<"); 25 | relationalOperators.insert("<="); 26 | logicalOperators.insert("AND"); 27 | logicalOperators.insert("OR"); 28 | } 29 | 30 | BinaryExpression* ExpressionParser::binaryExpression(std::string s){ 31 | if(s == "=") 32 | return new EqualExpression(); 33 | if(s == "<>") 34 | return new NotEqualExpression(); 35 | if(s == ">") 36 | return new GreaterThanExpression(); 37 | if(s == ">=") 38 | return new GreaterThanEqualExpression(); 39 | if(s == "<") 40 | return new LessThanExpression(); 41 | if(s == "<=") 42 | return new LessThanEqualExpression(); 43 | if(s == "AND") 44 | return new AndExpression(); 45 | if(s == "OR") 46 | return new OrExpression(); 47 | if(s == "+") 48 | return new AdditionExpression(); 49 | if(s == "-") 50 | return new SubtractionExpression(); 51 | if(s == "*") 52 | return new MultiplicationExpression(); 53 | if(s == "/") 54 | return new DivisionExpression(); 55 | raise_exception(); 56 | return NULL; 57 | } 58 | 59 | Expression* ExpressionParser::leafExpression(std::string s){ 60 | if(s[0] == '\'' && s[s.length()-1] == '\''){ 61 | char* ns = new char[s.length()-1]; 62 | std::copy(s.begin()+1, s.end()-1, ns); 63 | ns[s.length()-2] = '\0'; 64 | return new StringValueExpression(ns); 65 | //add starts with date 66 | //string expression 67 | }else if(isLong(s)){ 68 | return new LongValueExpression(extractLong(s)); 69 | }else if(isDouble(s)){ 70 | return new DoubleValueExpression(extractDouble(s)); 71 | }else{ 72 | return new ColExpression(s); 73 | } 74 | raise_exception(); 75 | return NULL; 76 | } 77 | 78 | Expression* ExpressionParser::parseExpression(std::vector tokens){ 79 | if(tokens.size() == 1) 80 | return leafExpression(tokens[0]); 81 | if(tokens.size() > 3 && tokens[tokens.size()-2] == "AS") 82 | return parseExpression(std::vector(tokens.begin(), tokens.end()-2)); 83 | for(int i = 0; i < tokens.size(); i++){ 84 | if(logicalOperators.find(tokens[i]) != logicalOperators.end()){ 85 | Expression *left = parseExpression(std::vector(tokens.begin(), tokens.begin()+i)); 86 | Expression *right = parseExpression(std::vector(tokens.begin()+i+1, tokens.end())); 87 | BinaryExpression *op = NULL; 88 | op = binaryExpression(tokens[i]); 89 | op->setLeftExpression(left); 90 | op->setRightExpression(right); 91 | return op; 92 | } 93 | } 94 | for(int i = 0; i < tokens.size(); i++){ 95 | if(relationalOperators.find(tokens[i]) != relationalOperators.end()){ 96 | Expression *left = parseExpression(std::vector(tokens.begin(), tokens.begin()+i)); 97 | Expression *right = parseExpression(std::vector(tokens.begin()+i+1, tokens.end())); 98 | BinaryExpression *op = NULL; 99 | op = binaryExpression(tokens[i]); 100 | op->setLeftExpression(left); 101 | op->setRightExpression(right); 102 | return op; 103 | } 104 | } 105 | for(int i = 0; i < tokens.size(); i++){ 106 | if(arithmeticOperators.find(tokens[i]) != arithmeticOperators.end()){ 107 | Expression *left = parseExpression(std::vector(tokens.begin(), tokens.begin()+i)); 108 | Expression *right = parseExpression(std::vector(tokens.begin()+i+1, tokens.end())); 109 | BinaryExpression *op = NULL; 110 | op = binaryExpression(tokens[i]); 111 | op->setLeftExpression(left); 112 | op->setRightExpression(right); 113 | return op; 114 | } 115 | } 116 | raise_exception(); 117 | return NULL; 118 | } 119 | 120 | std::vector ExpressionParser::parse(std::string exp){ 121 | std::vector tokens = splitString(exp, ','); 122 | std::vector exps; 123 | for(int i = 0; i < tokens.size(); i++){ 124 | if(tokens[i] == "") 125 | continue; 126 | //trim expression string 127 | std::vector expTokens = splitString(exp, ' '); 128 | Expression* e = parseExpression(expTokens); 129 | exps.push_back(e); 130 | } 131 | return exps; 132 | } 133 | 134 | //Shift to a utilities class 135 | std::vector ExpressionParser::splitString(const std::string s, char delimiter) { 136 | std::vector elems; 137 | std::stringstream ss(s); 138 | std::string item; 139 | while (std::getline(ss, item, delimiter)) 140 | elems.push_back(item); 141 | return elems; 142 | } 143 | 144 | bool ExpressionParser::isDouble(std::string s){ 145 | char* endptr = 0; 146 | std::strtod(s.c_str(), &endptr); 147 | if(*endptr != '\0' || endptr == s) 148 | return false; 149 | return true; 150 | } 151 | 152 | bool ExpressionParser::isLong(std::string s){ 153 | for(int i = 0; i < s.length(); i++) 154 | if(s[i] < '0' || s[i] > '9') 155 | return false; 156 | return true; 157 | } 158 | 159 | bool ExpressionParser::isDate(std::string s){ 160 | //TODO 161 | return true; 162 | } 163 | 164 | bool ExpressionParser::isString(std::string s){ 165 | return s[0] == '\'' && s[s.length()-1] == '\''; 166 | } 167 | 168 | long ExpressionParser::extractLong(std::string s){ 169 | return std::atoi(s.c_str()); 170 | } 171 | 172 | double ExpressionParser::extractDouble(std::string s){ 173 | return std::atof(s.c_str()); 174 | } 175 | -------------------------------------------------------------------------------- /Valkyrie/src/JoinOperator.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by vinayak on 12/7/15. 3 | // 4 | 5 | #include "../include/JoinOperator.h" 6 | #include "../include/ExpressionParser.h" 7 | #include "../include/Codegen.h" 8 | 9 | using namespace valkyrie; 10 | 11 | JoinOperator::JoinOperator(std::vector expressionList, std::vector children): Operator(children){ 12 | this->type = "JOIN"; 13 | this->expressions = expressionList; 14 | Schema *lsch = children[0]->getSchema(), *rsch = children[1]->getSchema(); 15 | ExpressionParser parser; 16 | joinClause = parser.parse(expressions[0])[0]; 17 | 18 | updateExpression(joinClause, lsch->getColumnMap(), rsch->getColumnMap(), lsch->getTableName(), rsch->getTableName()); 19 | schema = mergeSchemas(lsch, rsch); 20 | } 21 | 22 | void JoinOperator::consume() { 23 | switch(status) { 24 | case 0: 25 | status = 1; 26 | codegen::joinLeftConsume(this); 27 | break; 28 | case 1: 29 | status = 2; 30 | codegen::joinRightConsume(this); 31 | break; 32 | default: 33 | cout << "Unknown status: " << status << endl; 34 | exit(-1); 35 | } 36 | } 37 | 38 | void JoinOperator::produce() { 39 | children[0]->produce(); 40 | children[1]->produce(); 41 | 42 | Schema *lsch = children[0]->getSchema(), *rsch = children[1]->getSchema(); 43 | unordered_map colMap = schema->getColumnMap(); 44 | for(unordered_map::iterator it = colMap.begin(); it != colMap.end(); it++) 45 | updateExpression(it->second, lsch->getColumnMap(), rsch->getColumnMap(), lsch->getTableName(), rsch->getTableName()); 46 | codegen::joinConsume(*schema, parent); 47 | } 48 | 49 | void JoinOperator::updateExpression(Expression *exp, unordered_map lm, unordered_map rm 50 | , string ltable, string rtable) { 51 | ExprType t = exp->getType(); 52 | if(t != ExprType::COLEXPRESSION && t != ExprType::DOUBLEVALUEEXPRESSION && t != ExprType::STRINGVALUEEXPRESSION && 53 | t != ExprType::LONGVALUEEXPRESSION && t != ExprType::DATEVALUEEXPRESSION){ 54 | BinaryExpression* bexp = (BinaryExpression*)exp; 55 | updateExpression(bexp->getLeftExpression(), lm, rm, ltable, rtable); 56 | updateExpression(bexp->getRightExpression(), lm, rm, ltable, rtable); 57 | } else if(t == COLEXPRESSION){ 58 | ColExpression* col = (ColExpression*)exp; 59 | if(lm.find(col->getQualifiedName()) != lm.end()){ 60 | ColExpression* e = (ColExpression*)lm[col->getQualifiedName()]; 61 | col->setType(e->getDataType()); 62 | col->setColPos(e->getColPos()); 63 | left.push_back(col); 64 | } else if(rm.find(col->getQualifiedName()) != rm.end()){ 65 | ColExpression* e = (ColExpression*)rm[col->getQualifiedName()]; 66 | col->setType(e->getDataType()); 67 | col->setColPos(e->getColPos() + (int)lm.size()); 68 | ColExpression *rcol = new ColExpression(col->getQualifiedName(), col->getColPos()-(int)lm.size(), col->getDataType()); 69 | right.push_back(rcol); 70 | } else { 71 | std::cout << "column : " << col->getQualifiedName() << " not found in any schema " << std::endl; 72 | } 73 | } 74 | } 75 | 76 | Schema* JoinOperator::mergeSchemas(Schema *lsch, Schema *rsch) { 77 | Schema *res = new Schema("JOIN_" + lsch->getTableName() + "_" + rsch->getTableName()); 78 | std::vector attr = lsch->getAttributes(); 79 | for(int i = 0; i < attr.size(); i++) 80 | res->addAttribute(attr[i], lsch->getAttributeType(attr[i]), lsch->getAttrExpression(attr[i])); 81 | attr = rsch->getAttributes(); 82 | for(int i = 0; i < attr.size(); i++) 83 | res->addAttribute(attr[i], rsch->getAttributeType(attr[i]), rsch->getAttrExpression(attr[i])); 84 | return res; 85 | } 86 | 87 | vector JoinOperator::getLeftJoinAttrs() { 88 | return left; 89 | } 90 | 91 | vector JoinOperator::getRightJoinAttrs() { 92 | return right; 93 | } 94 | -------------------------------------------------------------------------------- /Valkyrie/src/Main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "../include/Parser.h" 5 | #include "../include/Codegen.h" 6 | 7 | using namespace std; 8 | 9 | 10 | /** 11 | * This is the starting point of the llvm generating phase. 12 | * This will take as input, the query plan from Java land, 13 | * and output the llvm code. 14 | */ 15 | int main(int argc, char** argv) 16 | { 17 | if(argc == 2 && strcmp(*argv, "-nollvm")) { 18 | codegen::nollvm(); 19 | } 20 | //Parsing 21 | string json; 22 | getline(cin, json); 23 | 24 | valkyrie::Parser parser; 25 | valkyrie::Operator *root = parser.parseJson(json); 26 | cout << root->queryPlan() << endl; 27 | 28 | /* Initializations */ 29 | codegen::initialize("LLVM"); 30 | 31 | /* Generate LLVM */ 32 | assert(root != NULL); 33 | root->produce(); 34 | 35 | /* Compile the generated module */ 36 | ExecutionEngine *engine = codegen::compile(); 37 | cout << "\n\nCompilation successful, now executing...\n\n"; 38 | 39 | /* Get a handle to the llvm function */ 40 | int (*executePlan)() = (int (*)())(engine->getPointerToNamedFunction("llvmStart", true)); 41 | 42 | /* Execute it */ 43 | executePlan(); 44 | } -------------------------------------------------------------------------------- /Valkyrie/src/Operator.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/Operator.h" 2 | 3 | using namespace valkyrie; 4 | 5 | Operator::Operator(std::vector children){ 6 | this->children = children; 7 | for(int i = 0; i < children.size(); i++) 8 | children[i]->setParent(this); 9 | } 10 | 11 | Operator::~Operator(){ 12 | delete schema; 13 | } 14 | 15 | std::vector Operator::getChildren(){ 16 | return this->children; 17 | } 18 | 19 | std::string Operator::toString(){ 20 | std::string exps = ""; 21 | for(int i = 0; i < expressions.size(); i++){ 22 | exps += expressions[i]; 23 | if(i != expressions.size()-1) 24 | exps += ", "; 25 | } 26 | return this->type + "[" + exps + "]"; 27 | } 28 | 29 | std::string Operator::queryPlan(){ 30 | std::string res = ""; 31 | std::stack > s; 32 | s.push(std::make_pair(this, 0)); 33 | while(!s.empty()){ 34 | std::pair p = s.top(); 35 | s.pop(); 36 | res += "\n"; 37 | for(int i = 0; i < p.second; i++) 38 | res += "\t"; 39 | res += p.first->toString(); 40 | std::vector v = p.first->getChildren(); 41 | for(int i = 0; i < v.size(); i++) 42 | s.push(std::make_pair(v[i], p.second+1)); 43 | } 44 | return res; 45 | } 46 | 47 | DataType Operator::mapType(std::string type){ 48 | if(type == "int" || type == "integer") 49 | return LONG; 50 | if(type == "char" || type == "varchar") 51 | return STRING; 52 | //check this 53 | if(type == "date") 54 | return DATE; 55 | if(type == "double" || type == "decimal") 56 | return DOUBLE; 57 | std::cerr << "Error parsing type" << std::endl; 58 | std::exit(-1); 59 | } 60 | 61 | Schema* Operator::getSchema(){ 62 | return schema; 63 | } 64 | 65 | void Operator::setParent(Operator* parent){ 66 | this->parent = parent; 67 | } 68 | -------------------------------------------------------------------------------- /Valkyrie/src/Parser.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/Parser.h" 2 | 3 | using namespace valkyrie; 4 | 5 | Operator* Parser::parseJson(std::string json){ 6 | rapidjson::Document doc; 7 | doc.Parse(json.c_str()); 8 | Operator *tree = createTree(doc["SRC"]); 9 | std::vector src; 10 | src.push_back(tree); 11 | Operator *root = new PrintOperator(std::vector(), src); 12 | return root; 13 | } 14 | 15 | Operator* Parser::createTree(const rapidjson::Value& node){ 16 | if(node == NULL) 17 | return NULL; 18 | std::string type = node["TYPE"].GetString(); 19 | std::vector expressions; 20 | const rapidjson::Value& expression = node["EXPRESSION"]; 21 | const rapidjson::Value& src = node["SRC"]; 22 | for(rapidjson::Value::ConstValueIterator it = expression.Begin(); it != expression.End(); it++) 23 | expressions.push_back(it->GetString()); 24 | std::vector children; 25 | for(rapidjson::Value::ConstValueIterator it = src.Begin(); it != src.End(); it++){ 26 | Operator *c = createTree(*it); 27 | children.push_back(c); 28 | } 29 | Operator *op = NULL; 30 | if(type == "SELECT"){ 31 | op = new SelectOperator(expressions, children); 32 | } else if(type == "PROJECT"){ 33 | op = new ProjectionOperator(expressions, children); 34 | } else if(type == "TABLE"){ 35 | op = new ScanOperator(expressions, children); 36 | } else if(type == "PRINT"){ 37 | op = new PrintOperator(expressions, children); 38 | } else if(type == "JOIN"){ 39 | op = new JoinOperator(expressions, children); 40 | } 41 | if(op == NULL){ 42 | std::cout << type << " Operator not implemented" << std::endl; 43 | exit(-1); 44 | } 45 | return op; 46 | } -------------------------------------------------------------------------------- /Valkyrie/src/PrintOperator.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/PrintOperator.h" 2 | #include "../include/Codegen.h" 3 | 4 | using namespace valkyrie; 5 | 6 | PrintOperator::PrintOperator(std::vector expressions, std::vector children) : Operator(children){ 7 | this->type = "PRINT"; 8 | this->expressions = expressions; 9 | schema = children[0]->getSchema(); 10 | types = (int *)&(schema->getTypes().front()); 11 | } 12 | 13 | void PrintOperator::produce(){ 14 | for(int i = 0; i < children.size(); i++) 15 | assert(children[i] != NULL); 16 | children[0]->produce(); 17 | } 18 | 19 | void PrintOperator::consume(){ 20 | codegen::printConsume(types); 21 | } 22 | -------------------------------------------------------------------------------- /Valkyrie/src/ProjectionOperator.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/ProjectionOperator.h" 2 | #include "../include/Codegen.h" 3 | #include "../include/ExpressionParser.h" 4 | #include "../include/Expression.h" 5 | 6 | using namespace valkyrie; 7 | 8 | ProjectionOperator::ProjectionOperator(std::vector expressionList, std::vector children) : Operator(children){ 9 | this->type = "PROJECT"; 10 | this->expressions = expressionList; 11 | schema = children[0]->getSchema(); 12 | ExpressionParser parser; 13 | bool allFound = false; 14 | vector attrs = schema->getAttributes(); 15 | for(int i = 0; i < expressions.size(); i++){ 16 | if(expressions[i] == "*"){ 17 | expressions.erase(expressions.begin()+i); 18 | if(!allFound) 19 | expressions.insert(expressions.begin()+i, attrs.begin(), attrs.end()); 20 | allFound = true; 21 | } 22 | } 23 | for(int i = 0; i < expressions.size(); i++){ 24 | std::vector exps = parser.parse(expressions[i]); 25 | projectionClauses.insert(projectionClauses.end(), exps.begin(), exps.end()); 26 | } 27 | } 28 | 29 | void ProjectionOperator::updateSchema(){ 30 | assert(projectionClauses.size() == expressions.size()); 31 | Schema *old = codegen::getSchema(); 32 | Schema *schema = new Schema(old->getTableName()); 33 | schema->setTuples(old->getTuples()); 34 | for(int i = 0; i < expressions.size(); i++){ 35 | std::size_t pos = expressions[i].find(" AS "); 36 | //TODO check the random column name assignment 37 | std::string colName = pos == std::string::npos ? "default_" + std::to_string(i) : expressions[i].substr(0, pos); 38 | updateExpression(projectionClauses[i], old->getColumnMap(), old->getTableName()); 39 | schema->addAttribute(colName, projectionClauses[i]->getDataType(), projectionClauses[i]); 40 | } 41 | codegen::setSchema(schema); 42 | } 43 | 44 | void ProjectionOperator::updateExpression(Expression *newExp, unordered_map m, string tableName) { 45 | ExprType t = newExp->getType(); 46 | if(t != ExprType::COLEXPRESSION && t != ExprType::DOUBLEVALUEEXPRESSION && t != ExprType::STRINGVALUEEXPRESSION && 47 | t != ExprType::LONGVALUEEXPRESSION && t != ExprType::DATEVALUEEXPRESSION){ 48 | BinaryExpression* b = (BinaryExpression*)newExp; 49 | updateExpression(b->getLeftExpression(), m, tableName); 50 | updateExpression(b->getRightExpression(), m, tableName); 51 | } else if(t == ExprType::COLEXPRESSION){ 52 | if(((ColExpression*)newExp)->getTableName() == "") 53 | ((ColExpression*)newExp)->setTableName(tableName); 54 | ColExpression* col = (ColExpression*)m[((ColExpression*)newExp)->getQualifiedName()]; 55 | ((ColExpression*)newExp)->setColPos(col->getColPos()); 56 | ((ColExpression*)newExp)->setType(col->getDataType()); 57 | } 58 | } 59 | 60 | void ProjectionOperator::produce(){ 61 | children[0]->produce(); 62 | } 63 | 64 | void ProjectionOperator::consume(){ 65 | updateSchema(); 66 | parent->consume(); 67 | } -------------------------------------------------------------------------------- /Valkyrie/src/ScanOperator.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/ScanOperator.h" 2 | #include "../include/Codegen.h" 3 | #include 4 | 5 | using namespace valkyrie; 6 | 7 | ScanOperator::ScanOperator(std::vector expressions, std::vector children) : Operator(children){ 8 | this->type = "TABLE"; 9 | this->expressions = expressions; 10 | convertToSchema(); 11 | } 12 | 13 | void ScanOperator::convertToSchema(){ 14 | std::string delimiter = "="; 15 | std::size_t t = expressions[0].find(delimiter); 16 | std::string tableName = expressions[0].substr(0, t); 17 | std::string tablePath = expressions[0].substr(t+1); 18 | schema = new Schema(tableName, tablePath); 19 | for(int i = 1; i < expressions.size(); i++){ 20 | t = expressions[i].find(delimiter); 21 | std::string colName = expressions[i].substr(0, t); 22 | std::string colType = expressions[i].substr(t+1); 23 | schema->addAttribute(colName, mapType(colType)); 24 | } 25 | } 26 | 27 | void ScanOperator::produce(){ 28 | for(int i = 0; i < children.size(); i++) 29 | assert(children[i] == NULL); 30 | assert(parent != NULL); 31 | consume(); 32 | } 33 | 34 | void ScanOperator::consume(){ 35 | schema->init(); 36 | //schema->dump(); 37 | codegen::scanConsume(*schema, parent); 38 | } -------------------------------------------------------------------------------- /Valkyrie/src/Schema.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "../include/Schema.h" 8 | #include "../include/Operator.h" 9 | #include 10 | 11 | using namespace std; 12 | using namespace valkyrie; 13 | 14 | string Schema::attrsVecsToCommaSepString(const vector& attr, const vector& types) const{ 15 | 16 | string line = "", tStr = ""; 17 | auto a = attr.begin(); 18 | auto t = types.begin(); 19 | 20 | while(a != attr.end() && t != types.end()) 21 | { 22 | switch(t.operator*()) { 23 | case LONG: 24 | tStr = "LONG"; 25 | break; 26 | case DOUBLE: 27 | tStr = "DOUBLE"; 28 | break; 29 | case STRING: 30 | tStr = "STRING"; 31 | break; 32 | case DATE: 33 | tStr = "DATE"; 34 | break; 35 | } 36 | line.append(a.operator*()).append(" "); 37 | line.append(tStr).append(", "); 38 | a++; 39 | t++; 40 | } 41 | 42 | return line; 43 | } 44 | 45 | Schema::Schema(string tablename) { 46 | setTableName(tablename); 47 | infile = NULL; 48 | } 49 | 50 | Schema::Schema(string tablename, string datafile) : datafile(datafile) { 51 | setTableName(tablename); 52 | infile = NULL; 53 | } 54 | 55 | void Schema::init() { 56 | if(infile != NULL) { 57 | close(); 58 | } 59 | 60 | assert(infile == NULL); 61 | 62 | infile = new ifstream(datafile); 63 | 64 | if(!infile) { 65 | cout << "Error, datafile invalid" << endl; 66 | exit(-1); 67 | } 68 | 69 | tuples.reserve(VDB_BLOCK_SIZE); 70 | } 71 | 72 | uint64_t Schema::loadBlock() { 73 | 74 | assert(infile != NULL); 75 | 76 | delete[] tuples[0]; 77 | tuples.clear(); 78 | 79 | assert(tuples.size() == 0); 80 | 81 | string line; 82 | uint64_t k = 0; 83 | 84 | unsigned long n = attributes.size(); 85 | LeafValue *theap = new LeafValue[VDB_BLOCK_SIZE * n]; 86 | 87 | while(getline(*infile, line)) { 88 | const char *l = line.c_str(); 89 | size_t lineptr = 0; 90 | 91 | for(int i=0; i= VDB_BLOCK_SIZE) break; 118 | } 119 | 120 | return k; 121 | } 122 | 123 | uint64_t Schema::close() { 124 | if(infile != NULL) { 125 | infile->close(); 126 | delete(infile); 127 | infile = NULL; 128 | } 129 | 130 | if(tuples.size() > 0) { 131 | if(block_allocated) { 132 | delete[] tuples[0]; 133 | } else { 134 | for(auto i : tuples) { 135 | delete(i); 136 | } 137 | } 138 | tuples.clear(); 139 | } 140 | 141 | return 0; 142 | } 143 | 144 | void Schema::addAttribute(string colName, DataType t) { 145 | string attr = formatAttrName(colName); 146 | attributes.push_back(attr); 147 | types.push_back(t); 148 | colMap.insert(make_pair(attr, new ColExpression(attr, attributes.size()-1, t))); 149 | } 150 | 151 | void Schema::addAttribute(string colName, DataType t, Expression *expression) { 152 | string attr = formatAttrName(colName); 153 | attributes.push_back(attr); 154 | types.push_back(t); 155 | colMap.insert(make_pair(attr, expression)); 156 | } 157 | 158 | 159 | 160 | ostream& operator<<(ostream &stream, const Schema &schema) { 161 | string attrs = schema.attrsVecsToCommaSepString(schema.getAttributes(), schema.getTypes()); 162 | return stream << schema.getTableName() << " : " << schema.getDataFile() << endl << attrs; 163 | } 164 | 165 | void Schema::setTuples(vector tuples) { 166 | this->tuples = tuples; 167 | } 168 | 169 | vector Schema::getTuples() { 170 | return this->tuples; 171 | } 172 | 173 | uint64_t Schema::getTuplePtr() const { 174 | return (uint64_t) &tuples[0]; 175 | } 176 | 177 | const vector Schema::getTypes() const{ 178 | return types; 179 | } 180 | 181 | std::string Schema::getDataFile() const{ 182 | return datafile; 183 | } 184 | 185 | vector Schema::getAttributes() const{ 186 | return attributes; 187 | } 188 | 189 | void Schema::setTableName(string tablename) { 190 | std::transform(tablename.begin(), tablename.end(), tablename.begin(), ::tolower); 191 | this->tablename = tablename; 192 | } 193 | 194 | size_t Schema::getAttributePos(string attr) const { 195 | string colName = formatAttrName(attr); 196 | for(size_t i =0; i < attributes.size(); i++){ 197 | if(attributes.at(i) == colName){ 198 | return i; 199 | } 200 | } 201 | std::cerr << "colName " << colName << " NOT Found" < Schema::getColumnMap(){ 225 | return this->colMap; 226 | }; 227 | 228 | Expression *Schema::getAttrExpression(string attr) { 229 | string colName = formatAttrName(attr); 230 | if(colMap.find(colName) == colMap.end()) 231 | return NULL; 232 | return colMap[colName]; 233 | } 234 | 235 | void Schema::addTuple(LeafValue *lv) { 236 | block_allocated = false; 237 | tuples.push_back(lv); 238 | } 239 | 240 | string Schema::formatAttrName(string name) const { 241 | //Check if the column name has table name with it, if not, then return qualified name 242 | if(name.find(".") == string::npos) 243 | return tablename + "." + name; 244 | return name; 245 | } 246 | -------------------------------------------------------------------------------- /Valkyrie/src/SelectOperator.cpp: -------------------------------------------------------------------------------- 1 | #include "../include/SelectOperator.h" 2 | #include "../include/Codegen.h" 3 | 4 | using namespace valkyrie; 5 | 6 | SelectOperator::SelectOperator(std::vector expressions, std::vector children) : Operator( 7 | children) { 8 | this->type = "SELECT"; 9 | this->expressions = expressions; 10 | ExpressionParser parser; 11 | selectClause = parser.parse(expressions[0])[0]; 12 | schema = children[0]->getSchema(); 13 | updateExpression(selectClause, schema->getColumnMap(), schema->getTableName()); 14 | } 15 | 16 | void SelectOperator::updateExpression(Expression *newExp, unordered_map m, std::string tableName) { 17 | ExprType t = newExp->getType(); 18 | if (t != ExprType::COLEXPRESSION && t != ExprType::DOUBLEVALUEEXPRESSION && t != ExprType::STRINGVALUEEXPRESSION && 19 | t != ExprType::LONGVALUEEXPRESSION && t != ExprType::DATEVALUEEXPRESSION) { 20 | BinaryExpression *b = (BinaryExpression *) newExp; 21 | updateExpression(b->getLeftExpression(), m, tableName); 22 | updateExpression(b->getRightExpression(), m, tableName); 23 | } else if (t == ExprType::COLEXPRESSION) { 24 | if(((ColExpression*)newExp)->getTableName() == "") 25 | ((ColExpression*)newExp)->setTableName(tableName); 26 | ColExpression *col = (ColExpression *) m[((ColExpression *) newExp)->getQualifiedName()]; 27 | ((ColExpression *) newExp)->setColPos(col->getColPos()); 28 | ((ColExpression *) newExp)->setType(col->getDataType()); 29 | } 30 | } 31 | 32 | void SelectOperator::produce() { 33 | assert(children[0] != NULL); 34 | children[0]->produce(); 35 | } 36 | 37 | void SelectOperator::consume() { 38 | assert(parent != NULL); 39 | codegen::selectConsume(selectClause, parent); 40 | } -------------------------------------------------------------------------------- /ValkyrieDB.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | rm -f bin/temp.json 3 | java -jar ./bin/MushroomCloud.jar --data ./test/data ./test/sql/tpch_schemas.sql $1 > bin/temp.json 4 | ./bin/llvmruntime < bin/temp.json -------------------------------------------------------------------------------- /allexps.sh: -------------------------------------------------------------------------------- 1 | echo "Setting up 100MB data" 2 | 3 | ./test/tpch-data-setup.py 0.1 4 | 5 | echo "Data setup complete" 6 | 7 | rm -rf small 8 | mkdir small 9 | 10 | echo "Running 1.sql [SMALL]" 11 | ./exp.sh test/sql/testcases/1.sql &>small/t1.out; 12 | 13 | echo "Running 2.sql [SMALL]" 14 | ./exp.sh test/sql/testcases/2.sql &>small/t2.out; 15 | 16 | echo "Running 3.sql [SMALL]" 17 | ./exp.sh test/sql/testcases/3.sql &>small/t3.out; 18 | 19 | echo "Running 4.sql [SMALL]" 20 | ./exp.sh test/sql/testcases/4.sql &>small/t4.out; 21 | 22 | echo "Running 5.sql [SMALL]" 23 | ./exp.sh test/sql/testcases/5.sql &>small/t5.out; 24 | 25 | echo "Setting up 1GB data" 26 | 27 | ./test/tpch-data-setup.py 1 28 | 29 | echo "Data setup complete" 30 | 31 | rm -rf large 32 | mkdir large 33 | 34 | 35 | echo "Running 1.sql [LARGE]" 36 | ./exp.sh test/sql/testcases/1.sql &>large/t1.out; 37 | 38 | echo "Running 2.sql [LARGE]" 39 | ./exp.sh test/sql/testcases/2.sql &>large/t2.out; 40 | 41 | echo "Running 3.sql [LARGE]" 42 | ./exp.sh test/sql/testcases/3.sql &>large/t3.out; 43 | 44 | echo "Running 4.sql [LARGE]" 45 | ./exp.sh test/sql/testcases/4.sql &>large/t4.out; 46 | 47 | echo "Running 5.sql [LARGE]" 48 | ./exp.sh test/sql/testcases/5.sql &>large/t5.out; -------------------------------------------------------------------------------- /exp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "SQLITE" 3 | time sqlite3 test/tpch.db < $1 > /dev/null 4 | 5 | echo "VALKYRIE" 6 | rm -f bin/temp.json 7 | java -jar ./bin/MushroomCloud.jar --data ./test/data ./test/sql/tpch_schemas.sql $1 > bin/temp.json 8 | time ./bin/llvmruntime -nollvm < bin/temp.json > /dev/null 9 | 10 | echo "MUSHROOM-CLOUD (PROJECTION AWARE SCAN)" 11 | time java -jar bin/MushroomCloud.jar --data test/data/ --exec --pdp test/sql/tpch_schemas.sql $1 > /dev/null 12 | 13 | echo "MUSHROOM-CLOUD" 14 | time java -jar bin/MushroomCloud.jar --data test/data/ --exec test/sql/tpch_schemas.sql $1 > /dev/null 15 | -------------------------------------------------------------------------------- /test/cprogs/q1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Legacy25/ValkyrieDB/c2adee18658658e78b7a4b464c8db91657fe58c1/test/cprogs/q1 -------------------------------------------------------------------------------- /test/cprogs/q1.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void handle_error(char* err_mesg, int exit_code) { 6 | printf("%s\n", err_mesg); 7 | exit(exit_code); 8 | } 9 | 10 | int main(int argc, char **argv) { 11 | if (argc != 2) { 12 | handle_error("Usage: q1 ", -1); 13 | } 14 | 15 | char* tablename = "lineitem"; 16 | char* filename = malloc(strlen(argv[1]) + strlen(tablename) + 6); 17 | sprintf(filename, "%s%s.tbl", argv[1], tablename); 18 | 19 | FILE *f = fopen(filename, "r"); 20 | if(!f) { 21 | handle_error("Couldn't open file", -1); 22 | } 23 | 24 | long ok, pk, sk, ln, q; 25 | double ep, disc, tax; 26 | char rf[1], ls[1], sd[10], cd[10], rd[10], sm[10], si[25], com[44]; 27 | 28 | while(!feof(f)) { 29 | fscanf(f, "%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|", 30 | &ok, &pk, &sk, &ln, &q, &ep, &disc, &tax, rf, ls, sd, cd, rd, sm, si, com); 31 | printf("%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%s|%s|%s|%s|%s|%s|%s|%s|\n", 32 | ok, pk, sk, ln, q, ep, disc, tax, rf, ls, sd, cd, rd, sm, si, com); 33 | } 34 | 35 | fclose(f); 36 | } -------------------------------------------------------------------------------- /test/cprogs/q2.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void handle_error(char* err_mesg, int exit_code) { 6 | printf("%s\n", err_mesg); 7 | exit(exit_code); 8 | } 9 | 10 | int main(int argc, char **argv) { 11 | if (argc != 2) { 12 | handle_error("Usage: q2 ", -1); 13 | } 14 | 15 | char* tablename = "lineitem"; 16 | char* filename = (char*)malloc(strlen(argv[1]) + strlen(tablename) + 6); 17 | sprintf(filename, "%s%s.tbl", argv[1], tablename); 18 | 19 | FILE *f = fopen(filename, "r"); 20 | if(!f) { 21 | handle_error("Couldn't open file", -1); 22 | } 23 | 24 | long ok, pk, sk, ln, q; 25 | double ep, disc, tax; 26 | char rf[1], ls[1], sd[10], cd[10], rd[10], sm[10], si[25], com[44]; 27 | 28 | while(!feof(f)) { 29 | fscanf(f, "%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|", 30 | &ok, &pk, &sk, &ln, &q, &ep, &disc, &tax, rf, ls, sd, cd, rd, sm, si, com); 31 | if(disc > 0.05 && disc < 0.07 && q < 24) 32 | printf("%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%s|%s|%s|%s|%s|%s|%s|%s|\n", 33 | ok, pk, sk, ln, q, ep, disc, tax, rf, ls, sd, cd, rd, sm, si, com); 34 | } 35 | 36 | fclose(f); 37 | } -------------------------------------------------------------------------------- /test/cprogs/q3.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void handle_error(char* err_mesg, int exit_code) { 6 | printf("%s\n", err_mesg); 7 | exit(exit_code); 8 | } 9 | 10 | int main(int argc, char **argv) { 11 | if (argc != 2) { 12 | handle_error("Usage: q3 ", -1); 13 | } 14 | 15 | char* tablename = "lineitem"; 16 | char* filename = (char*)malloc(strlen(argv[1]) + strlen(tablename) + 6); 17 | sprintf(filename, "%s%s.tbl", argv[1], tablename); 18 | 19 | FILE *f = fopen(filename, "r"); 20 | if(!f) { 21 | handle_error("Couldn't open file", -1); 22 | } 23 | 24 | long ok, pk, sk, ln, q; 25 | double ep, disc, tax; 26 | char rf[1], ls[1], sd[10], cd[10], rd[10], sm[10], si[25], com[44]; 27 | 28 | while(!feof(f)) { 29 | fscanf(f, "%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|", 30 | &ok, &pk, &sk, &ln, &q, &ep, &disc, &tax, rf, ls, sd, cd, rd, sm, si, com); 31 | if(disc > 0.05 && disc < 0.07 && q < 24) 32 | printf("%lf|%lf|\n", ep, disc); 33 | } 34 | 35 | fclose(f); 36 | } -------------------------------------------------------------------------------- /test/cprogs/q4.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | struct data{ 11 | int val; 12 | char* str; 13 | data(int val){ this->val = val; } 14 | data(char* val){ this->str = val; } 15 | }; 16 | 17 | void handle_error(char* err_mesg, int exit_code) { 18 | printf("%s\n", err_mesg); 19 | exit(exit_code); 20 | } 21 | 22 | int main(int argc, char **argv) { 23 | if (argc != 2) { 24 | handle_error("Usage: q4 ", -1); 25 | } 26 | 27 | char* table3 = "lineitem"; 28 | char* table1 = "customer"; 29 | char* table2 = "orders"; 30 | 31 | char* file1 = malloc(strlen(argv[1]) + strlen(table1) + 6); 32 | char* file2 = malloc(strlen(argv[1]) + strlen(table2) + 6); 33 | char* file3 = malloc(strlen(argv[1]) + strlen(table3) + 6); 34 | 35 | sprintf(filename, "%s%s.tbl", argv[1], tablename); 36 | 37 | FILE *f = fopen(file2, "r"); 38 | if(!f) { 39 | handle_error("Couldn't open file", -1); 40 | } 41 | 42 | unordered_map> join1; 43 | long orderKey, custKey, shippriority; 44 | double totalprice; 45 | char orderstatus, orderdate[10], orderpriority[15], clerk[15], comment[79]; 46 | while(!feof(f)){ 47 | fscanf(f, "%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|", 48 | &orderKey, &custKey, &orderstatus, &totalprice, orderdate, orderpriority, 49 | clerk, &shippriority, comment); 50 | if(totalprice > 460000.0){ 51 | vector row; 52 | row.push_back(data(orderKey)); 53 | row.push_back(data(custKey)); 54 | row.push_back(data(orderstatus)); 55 | row.push_back(data(totalprice)); 56 | row.push_back(data(orderdate)); 57 | row.push_back(data(orderpriority)); 58 | row.push_back(data(clerk)); 59 | row.push_back(data(shippriority)); 60 | join1.insert(make_pair>(orderKey, row)); 61 | } 62 | } 63 | 64 | long ok, pk, sk, ln, q; 65 | double ep, disc, tax; 66 | char rf[1], ls[1], sd[10], cd[10], rd[10], sm[10], si[25], com[44]; 67 | 68 | while(!feof(f)) { 69 | fscanf(f, "%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|%[^|]|", 70 | &ok, &pk, &sk, &ln, &q, &ep, &disc, &tax, rf, ls, sd, cd, rd, sm, si, com); 71 | printf("%ld|%ld|%ld|%ld|%ld|%lf|%lf|%lf|%s|%s|%s|%s|%s|%s|%s|%s|\n", 72 | ok, pk, sk, ln, q, ep, disc, tax, rf, ls, sd, cd, rd, sm, si, com); 73 | } 74 | 75 | fclose(f); 76 | } -------------------------------------------------------------------------------- /test/sql/ddl-schema.sql: -------------------------------------------------------------------------------- 1 | create table Part( 2 | partkey INTEGER PRIMARY KEY, name, mfgr, brand, type, size INTEGER, container, retailprice real, comment); 3 | 4 | create table Supplier( 5 | suppKey INTEGER PRIMARY KEY, name, address, nationkey INTEGER, phone, acctbal REAL, comment); 6 | 7 | create table PartSupp( 8 | partKey INTEGER, suppKey INTEGER, availqty INTEGER, supplycost REAL, comment); 9 | 10 | create table Customer(CustKey INTEGER PRIMARY KEY, name, address, nationkey INTEGER, phone, acctbal REAL, mktsegment, comment); 11 | 12 | create table Nation( 13 | nationkey INTEGER PRIMARY KEY, name, regionkey INTEGER, comment); 14 | 15 | create table Region( 16 | regionkey INTEGER PRIMARY KEY, name, comment); 17 | 18 | create table LineItem( 19 | orderKey INTEGER, partKey INTEGER, suppKey INTEGER, lineNumber INTEGER, quantity INTEGER, extendedPrice REAL, discount REAL, tax REAL, returnFlag, lineStatus, shipDate, commitDate, receiptDate, shipInstruct, shipMode, comment); 20 | 21 | create table Orders( 22 | orderKey INTEGER PRIMARY KEY, custKey INTEGER, orderStatus, totalPrice REAL, orderDate, orderPriority, clerk, shipPriority, comment); -------------------------------------------------------------------------------- /test/sql/test.sql: -------------------------------------------------------------------------------- 1 | /*SELECT 2 | lineitem.orderkey, 3 | sum(lineitem.extendedprice*(1-lineitem.discount)) as revenue, 4 | orders.orderdate, 5 | orders.shippriority 6 | FROM 7 | customer, 8 | orders, 9 | lineitem 10 | WHERE 11 | customer.mktsegment = 'BUILDING' and customer.custkey = orders.custkey 12 | and lineitem.orderkey = orders.orderkey 13 | and orders.orderdate < DATE('1995-03-15') 14 | and lineitem.shipdate > DATE('1995-03-15') 15 | GROUP BY lineitem.orderkey, orders.orderdate, orders.shippriority 16 | ORDER BY revenue desc, orders.orderdate;*/ 17 | 18 | -- select * from lineitem where orderkey < 200; 19 | -- select * from customer; 20 | -- select nation.name, region.name from nation, region where nation.regionkey = region.regionkey; 21 | -- select nation.name, region.name from nation, region where nation.regionkey = region.regionkey; 22 | 23 | -- SELECT 24 | -- orders.orderkey, 25 | -- orders.totalprice, 26 | -- lineitem.extendedprice, 27 | -- lineitem.discount, 28 | -- lineitem.quantity, 29 | -- customer.name 30 | -- FROM 31 | -- customer, 32 | -- orders, 33 | -- lineitem 34 | -- WHERE 35 | -- customer.custkey = orders.custkey 36 | -- and lineitem.orderkey = orders.orderkey 37 | -- and orders.totalprice > 460000.0; 38 | 39 | --select * from customer where name = 'Customer#000000016'; 40 | 41 | SELECT orders.orderkey, orders.totalprice, lineitem.extendedprice, lineitem.discount, lineitem.quantity, customer.name 42 | FROM customer, orders, lineitem 43 | WHERE 44 | customer.custkey = orders.custkey 45 | and lineitem.orderkey = orders.orderkey 46 | and orders.totalprice > 460000.0 47 | AND orders.orderdate >= '1994-01-01' 48 | AND orders.orderdate < '1995-01-01'; -------------------------------------------------------------------------------- /test/sql/testcases/1.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM LINEITEM; -------------------------------------------------------------------------------- /test/sql/testcases/2.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | from 4 | lineitem 5 | where discount>0.05 and discount<0.07 and quantity < 24; -------------------------------------------------------------------------------- /test/sql/testcases/3.sql: -------------------------------------------------------------------------------- 1 | select 2 | extendedprice, discount 3 | from 4 | lineitem 5 | where discount>0.05 and discount<0.07 and quantity < 24; 6 | 7 | -------------------------------------------------------------------------------- /test/sql/testcases/4.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | orders.orderkey, 3 | orders.totalprice, 4 | lineitem.extendedprice, 5 | lineitem.discount, 6 | lineitem.quantity, 7 | customer.name 8 | FROM 9 | customer, 10 | orders, 11 | lineitem 12 | WHERE 13 | customer.custkey = orders.custkey 14 | and lineitem.orderkey = orders.orderkey 15 | and orders.totalprice > 460000.0; 16 | -------------------------------------------------------------------------------- /test/sql/testcases/5.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | nation.name, 3 | lineitem.extendedprice, 4 | lineitem.discount 5 | FROM 6 | orders, 7 | lineitem, 8 | supplier, 9 | nation, 10 | region 11 | WHERE 12 | lineitem.orderkey = orders.orderkey 13 | AND supplier.nationkey = nation.nationkey 14 | AND nation.regionkey = region.regionkey 15 | AND region.name = 'ASIA' 16 | AND orders.orderdate >= '1994-01-01' 17 | AND orders.orderdate < '1995-01-01' 18 | AND lineitem.suppkey = supplier.suppkey; -------------------------------------------------------------------------------- /test/sql/tpch1.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE LINEITEM ( 2 | orderkey INT REFERENCES ORDERS, 3 | partkey INT, 4 | suppkey INT REFERENCES SUPPLIERS, 5 | linenumber INT, 6 | quantity DECIMAL, 7 | extendedprice DECIMAL, 8 | discount DECIMAL, 9 | tax DECIMAL, 10 | returnflag CHAR(1), 11 | linestatus CHAR(1), 12 | shipdate DATE, 13 | commitdate DATE, 14 | receiptdate DATE, 15 | shipinstruct CHAR(25), 16 | shipmode CHAR(10), 17 | comment VARCHAR(44), 18 | PRIMARY KEY (orderkey, linenumber) 19 | ); 20 | CREATE TABLE ORDERS ( 21 | orderkey INT, 22 | custkey INT REFERENCES CUSTOMER, 23 | orderstatus CHAR(1), 24 | totalprice DECIMAL, 25 | orderdate DATE, 26 | orderpriority CHAR(15), 27 | clerk CHAR(15), 28 | shippriority INT, 29 | comment VARCHAR(79), 30 | PRIMARY KEY (orderkey) 31 | ); 32 | CREATE TABLE PART ( 33 | partkey INT, 34 | name VARCHAR(55), 35 | mfgr CHAR(25), 36 | brand CHAR(10), 37 | type VARCHAR(25), 38 | size INT, 39 | container CHAR(10), 40 | retailprice DECIMAL, 41 | comment VARCHAR(23), 42 | PRIMARY KEY (partkey) 43 | ); 44 | CREATE TABLE CUSTOMER ( 45 | custkey INT, 46 | name VARCHAR(25), 47 | address VARCHAR(40), 48 | nationkey INT REFERENCES NATION, 49 | phone CHAR(15), 50 | acctbal DECIMAL, 51 | mktsegment CHAR(10), 52 | comment VARCHAR(117), 53 | PRIMARY KEY (custkey) 54 | ); 55 | CREATE TABLE SUPPLIER ( 56 | suppkey INT, 57 | name CHAR(25), 58 | address VARCHAR(40), 59 | nationkey INT REFERENCES NATION, 60 | phone CHAR(15), 61 | acctbal DECIMAL, 62 | comment VARCHAR(101), 63 | PRIMARY KEY (suppkey) 64 | ); 65 | CREATE TABLE PARTSUPP ( 66 | partkey INT REFERENCES PART, 67 | suppkey INT REFERENCES SUPPLIER, 68 | availqty INT, 69 | supplycost DECIMAL, 70 | comment VARCHAR(199), 71 | PRIMARY KEY (partkey, suppkey) 72 | ); 73 | CREATE TABLE NATION ( 74 | nationkey INT, 75 | name CHAR(25), 76 | regionkey INT REFERENCES REGION, 77 | comment VARCHAR(152), 78 | PRIMARY KEY (nationkey) 79 | ); 80 | CREATE TABLE REGION ( 81 | regionkey INT, 82 | name CHAR(25), 83 | comment VARCHAR(152), 84 | PRIMARY KEY (regionkey) 85 | ); 86 | 87 | SELECT 88 | returnflag, 89 | linestatus, 90 | sum(quantity) as sum_qty, 91 | sum(extendedprice) as sum_base_price, 92 | sum(extendedprice*(1-discount)) as sum_disc_price, 93 | sum(extendedprice*(1-discount)*(1+tax)) as sum_charge, 94 | avg(quantity) as avg_qty, 95 | avg(extendedprice) as avg_price, 96 | avg(discount) as avg_disc, 97 | count(*) as count_order 98 | FROM 99 | lineitem 100 | WHERE 101 | shipdate <= DATE('1998-09-01') 102 | GROUP BY 103 | returnflag, linestatus 104 | ORDER BY 105 | returnflag, linestatus; 106 | -------------------------------------------------------------------------------- /test/sql/tpch10.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | customer.custkey, 3 | customer.name, 4 | customer.acctbal, 5 | nation.name, 6 | customer.address, 7 | customer.phone, 8 | customer.comment, 9 | lineitem.extendedprice, lineitem.discount 10 | FROM 11 | customer, 12 | orders, 13 | lineitem, 14 | nation 15 | WHERE 16 | customer.custkey = orders.custkey 17 | AND lineitem.orderkey = orders.orderkey 18 | AND customer.nationkey = nation.nationkey; -------------------------------------------------------------------------------- /test/sql/tpch12.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE LINEITEM ( 2 | orderkey INT REFERENCES ORDERS, 3 | partkey INT, 4 | suppkey INT REFERENCES SUPPLIERS, 5 | linenumber INT, 6 | quantity DECIMAL, 7 | extendedprice DECIMAL, 8 | discount DECIMAL, 9 | tax DECIMAL, 10 | returnflag CHAR(1), 11 | linestatus CHAR(1), 12 | shipdate DATE, 13 | commitdate DATE, 14 | receiptdate DATE, 15 | shipinstruct CHAR(25), 16 | shipmode CHAR(10), 17 | comment VARCHAR(44), 18 | PRIMARY KEY (orderkey, linenumber) 19 | ); 20 | CREATE TABLE ORDERS ( 21 | orderkey INT, 22 | custkey INT REFERENCES CUSTOMER, 23 | orderstatus CHAR(1), 24 | totalprice DECIMAL, 25 | orderdate DATE, 26 | orderpriority CHAR(15), 27 | clerk CHAR(15), 28 | shippriority INT, 29 | comment VARCHAR(79), 30 | PRIMARY KEY (orderkey) 31 | ); 32 | CREATE TABLE PART ( 33 | partkey INT, 34 | name VARCHAR(55), 35 | mfgr CHAR(25), 36 | brand CHAR(10), 37 | type VARCHAR(25), 38 | size INT, 39 | container CHAR(10), 40 | retailprice DECIMAL, 41 | comment VARCHAR(23), 42 | PRIMARY KEY (partkey) 43 | ); 44 | CREATE TABLE CUSTOMER ( 45 | custkey INT, 46 | name VARCHAR(25), 47 | address VARCHAR(40), 48 | nationkey INT REFERENCES NATION, 49 | phone CHAR(15), 50 | acctbal DECIMAL, 51 | mktsegment CHAR(10), 52 | comment VARCHAR(117), 53 | PRIMARY KEY (custkey) 54 | ); 55 | CREATE TABLE SUPPLIER ( 56 | suppkey INT, 57 | name CHAR(25), 58 | address VARCHAR(40), 59 | nationkey INT REFERENCES NATION, 60 | phone CHAR(15), 61 | acctbal DECIMAL, 62 | comment VARCHAR(101), 63 | PRIMARY KEY (suppkey) 64 | ); 65 | CREATE TABLE PARTSUPP ( 66 | partkey INT REFERENCES PART, 67 | suppkey INT REFERENCES SUPPLIER, 68 | availqty INT, 69 | supplycost DECIMAL, 70 | comment VARCHAR(199), 71 | PRIMARY KEY (partkey, suppkey) 72 | ); 73 | CREATE TABLE NATION ( 74 | nationkey INT, 75 | name CHAR(25), 76 | regionkey INT REFERENCES REGION, 77 | comment VARCHAR(152), 78 | PRIMARY KEY (nationkey) 79 | ); 80 | CREATE TABLE REGION ( 81 | regionkey INT, 82 | name CHAR(25), 83 | comment VARCHAR(152), 84 | PRIMARY KEY (regionkey) 85 | ); 86 | 87 | select lineitem.shipmode, 88 | sum(case when orders.orderpriority = '1-URGENT' 89 | or orders.orderpriority = '2-HIGH' 90 | then 1 91 | else 0 end) as high_line_count, 92 | sum(case when orders.orderpriority <> '1-URGENT' 93 | and orders.orderpriority <> '2-HIGH' 94 | then 1 95 | else 0 end) as low_line_count 96 | from lineitem, orders 97 | where orders.orderkey = lineitem.orderkey 98 | and (lineitem.shipmode='AIR' or lineitem.shipmode='MAIL' or lineitem.shipmode='TRUCK' or lineitem.shipmode='SHIP') 99 | and lineitem.commitdate < lineitem.receiptdate 100 | and lineitem.shipdate < lineitem.commitdate 101 | and lineitem.receiptdate >= date('1995-03-05') 102 | and lineitem.receiptdate < date('1996-03-05') 103 | group by lineitem.shipmode 104 | order by lineitem.shipmode; 105 | -------------------------------------------------------------------------------- /test/sql/tpch3.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | orders.orderkey, 3 | orders.totalprice, 4 | lineitem.extendedprice, 5 | lineitem.discount, 6 | lineitem.quantity, 7 | customer.name 8 | FROM 9 | customer, 10 | orders, 11 | lineitem 12 | WHERE 13 | customer.custkey = orders.custkey 14 | and lineitem.orderkey = orders.orderkey 15 | and orders.totalprice > 460000.0; 16 | -------------------------------------------------------------------------------- /test/sql/tpch5.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | nation.name, 3 | lineitem.extendedprice, lineitem.discount 4 | FROM 5 | region, 6 | nation, 7 | customer, 8 | orders, 9 | lineitem, 10 | supplier 11 | WHERE 12 | customer.custkey = orders.custkey 13 | and lineitem.orderkey = orders.orderkey 14 | and lineitem.suppkey = supplier.suppkey 15 | and customer.nationkey = supplier.nationkey 16 | and supplier.nationkey = nation.nationkey 17 | and nation.regionkey = region.regionkey; -------------------------------------------------------------------------------- /test/sql/tpch6.sql: -------------------------------------------------------------------------------- 1 | select 2 | extendedprice, discount 3 | from 4 | lineitem 5 | where discount>0.05 and discount<0.07 and quantity < 24; 6 | 7 | -------------------------------------------------------------------------------- /test/sql/tpch_schemas.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE LINEITEM ( 2 | orderkey INT REFERENCES ORDERS, 3 | partkey INT, 4 | suppkey INT REFERENCES SUPPLIERS, 5 | linenumber INT, 6 | quantity INT, 7 | extendedprice DECIMAL, 8 | discount DECIMAL, 9 | tax DECIMAL, 10 | returnflag CHAR(1), 11 | linestatus CHAR(1), 12 | shipdate DATE, 13 | commitdate DATE, 14 | receiptdate DATE, 15 | shipinstruct CHAR(25), 16 | shipmode CHAR(10), 17 | comment VARCHAR(44), 18 | PRIMARY KEY (orderkey, linenumber) 19 | ); 20 | CREATE TABLE ORDERS ( 21 | orderkey INT, 22 | custkey INT REFERENCES CUSTOMER, 23 | orderstatus CHAR(1), 24 | totalprice DECIMAL, 25 | orderdate DATE, 26 | orderpriority CHAR(15), 27 | clerk CHAR(15), 28 | shippriority INT, 29 | comment VARCHAR(79), 30 | PRIMARY KEY (orderkey) 31 | ); 32 | CREATE TABLE PART ( 33 | partkey INT, 34 | name VARCHAR(55), 35 | mfgr CHAR(25), 36 | brand CHAR(10), 37 | type VARCHAR(25), 38 | size INT, 39 | container CHAR(10), 40 | retailprice DECIMAL, 41 | comment VARCHAR(23), 42 | PRIMARY KEY (partkey) 43 | ); 44 | CREATE TABLE CUSTOMER ( 45 | custkey INT, 46 | name VARCHAR(25), 47 | address VARCHAR(40), 48 | nationkey INT REFERENCES NATION, 49 | phone CHAR(15), 50 | acctbal DECIMAL, 51 | mktsegment CHAR(10), 52 | comment VARCHAR(117), 53 | PRIMARY KEY (custkey) 54 | ); 55 | CREATE TABLE SUPPLIER ( 56 | suppkey INT, 57 | name CHAR(25), 58 | address VARCHAR(40), 59 | nationkey INT REFERENCES NATION, 60 | phone CHAR(15), 61 | acctbal DECIMAL, 62 | comment VARCHAR(101), 63 | PRIMARY KEY (suppkey) 64 | ); 65 | CREATE TABLE PARTSUPP ( 66 | partkey INT REFERENCES PART, 67 | suppkey INT REFERENCES SUPPLIER, 68 | availqty INT, 69 | supplycost DECIMAL, 70 | comment VARCHAR(199), 71 | PRIMARY KEY (partkey, suppkey) 72 | ); 73 | CREATE TABLE NATION ( 74 | nationkey INT, 75 | name CHAR(25), 76 | regionkey INT REFERENCES REGION, 77 | comment VARCHAR(152), 78 | PRIMARY KEY (nationkey) 79 | ); 80 | CREATE TABLE REGION ( 81 | regionkey INT, 82 | name CHAR(25), 83 | comment VARCHAR(152), 84 | PRIMARY KEY (regionkey) 85 | ); 86 | 87 | -------------------------------------------------------------------------------- /test/tpch-data-setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import sys 5 | import shutil 6 | import subprocess 7 | import glob 8 | import sqlite3 9 | 10 | home = os.path.dirname(os.path.realpath(__file__)) 11 | tpchgenrepo = "https://github.com/Legacy25/tpch-dbgen.git" 12 | tpchdb = "tpch.db" 13 | sf = sys.argv[1] 14 | 15 | def clonerepo(repo): 16 | os.chdir(home) 17 | if(os.path.exists("tpch-dbgen")): 18 | shutil.rmtree("tpch-dbgen") 19 | subprocess.call(["git", "clone", repo]) 20 | 21 | def initdatafolder(): 22 | os.chdir(home) 23 | if(os.path.exists("data")): 24 | shutil.rmtree("data") 25 | os.mkdir("data") 26 | os.chdir("tpch-dbgen") 27 | subprocess.call(["make"]) 28 | subprocess.call(["./dbgen", "-s", sf]) 29 | tblfiles = glob.glob("*.tbl") 30 | for f in tblfiles: 31 | os.rename(f, os.path.join("../data", f)) 32 | os.chdir("..") 33 | 34 | def createDB(dbname): 35 | if(os.path.exists(tpchdb)): 36 | os.remove(tpchdb) 37 | conn = sqlite3.connect(tpchdb) 38 | return conn.cursor() 39 | 40 | def initDB(cursor): 41 | f = open("sql/ddl-schema.sql") 42 | sql = f.read() 43 | stmts = sql.split(";") 44 | for s in stmts: 45 | cursor.execute(s) 46 | 47 | def loaddata(cursor): 48 | tblfiles = glob.glob("data/*.tbl") 49 | 50 | for f in tblfiles: 51 | arg = ".import {} {}".format(f, f.split("/")[1].split(".tbl")[0]) 52 | subprocess.call(["sqlite3", "tpch.db", arg]) 53 | 54 | def main(): 55 | clonerepo(tpchgenrepo) 56 | initdatafolder() 57 | cursor = createDB(tpchdb) 58 | initDB(cursor) 59 | loaddata(cursor) 60 | 61 | if __name__ == "__main__": 62 | main() --------------------------------------------------------------------------------