├── .bazelrc ├── .clang-format ├── .gitignore ├── BUILD ├── LICENSE ├── README.md ├── SQLParser ├── .gitignore ├── BUILD └── Sql.g4 ├── SimpleDB ├── .gitignore ├── BUILD ├── include │ └── SimpleDB │ │ ├── DBMS.h │ │ ├── Error.h │ │ ├── SimpleDB.h │ │ └── internal │ │ ├── CacheManager.h │ │ ├── Column.h │ │ ├── Comparer.h │ │ ├── FileCoordinator.h │ │ ├── FileDescriptor.h │ │ ├── FileManager.h │ │ ├── Index.h │ │ ├── IndexedTable.h │ │ ├── JoinedTable.h │ │ ├── LinkedList.h │ │ ├── Logger.h │ │ ├── Macros.h │ │ ├── PageFile.h │ │ ├── PageHandle.h │ │ ├── ParseHelper.h │ │ ├── ParseTreeVisitor.h │ │ ├── QueryBuilder.h │ │ ├── QueryDataSource.h │ │ ├── QueryFilter.h │ │ ├── Service.h │ │ └── Table.h └── src │ ├── backend │ ├── Index.cc │ ├── IndexedTable.cc │ ├── JoinedTable.cc │ ├── QueryBuilder.cc │ ├── QueryFilter.cc │ └── Table.cc │ ├── dbms │ ├── DBMS.cc │ ├── ParseTreeVisitor.cc │ └── SystemTableSchema.cc │ ├── io │ ├── CacheManager.cc │ ├── FileCoordinator.cc │ └── FileManager.cc │ └── util │ └── Logger.cc ├── SimpleDBClient ├── .gitignore ├── BUILD ├── client.py ├── main.py ├── requirements.txt └── util.py ├── SimpleDBServer ├── BUILD └── src │ ├── SQLService.cc │ ├── SQLService.h │ └── main.cc ├── SimpleDBService ├── BUILD └── query.proto ├── SimpleDBTest ├── .gitignore ├── BUILD ├── CacheManagerTest.cc ├── DBMSTest.cc ├── FileCoordinatorTest.cc ├── FileManagerTest.cc ├── IndexTest.cc ├── IndexedTableTest.cc ├── ParameterTest.cc ├── QueryConditionTest.cc ├── TableTest.cc ├── Util.cc ├── Util.h └── main.cc ├── WORKSPACE └── docs ├── client_screenshot.png ├── dep-graph.svg ├── design.md └── server_screenshot.png /.bazelrc: -------------------------------------------------------------------------------- 1 | build --cxxopt='-std=c++17' 2 | build:macos --linkopt='-Wl,-no_fixup_chains' 3 | build:linux --linkopt='-fuse-ld=lld' 4 | build --host_action_env=PATH=/bin:/usr/bin --incompatible_strict_action_env 5 | build --java_runtime_version=remotejdk_11 6 | build --client_env=CC=clang 7 | 8 | test --test_output=all -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | { BasedOnStyle: Google, IndentWidth: 4, ColumnLimit: 80, AccessModifierOffset: -4 } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .DS_Store 3 | dbdata*/ 4 | tmp/ 5 | 6 | # Bazel 7 | # Ignore backup files. 8 | *~ 9 | # Ignore Vim swap files. 10 | .*.swp 11 | # Ignore files generated by IDEs. 12 | /.aswb/ 13 | /.cache/ 14 | /.classpath 15 | /.clwb/ 16 | /.factorypath 17 | /.idea/ 18 | /.ijwb/ 19 | /.project 20 | /.settings 21 | /.vscode/ 22 | /bazel.iml 23 | # Ignore all bazel-* symlinks. There is no full list since this can change 24 | # based on the name of the directory bazel is cloned into. 25 | /bazel-* 26 | # Ignore outputs generated during Bazel bootstrapping. 27 | /output/ 28 | # Ignore jekyll build output. 29 | /production 30 | /.sass-cache 31 | # Bazelisk version file 32 | .bazelversion 33 | # User-specific .bazelrc 34 | user.bazelrc 35 | 36 | # Prerequisites 37 | *.d 38 | 39 | # Compiled Object files 40 | *.slo 41 | *.lo 42 | *.o 43 | *.obj 44 | 45 | # Precompiled Headers 46 | *.gch 47 | *.pch 48 | 49 | # Compiled Dynamic libraries 50 | *.so 51 | *.dylib 52 | *.dll 53 | 54 | # Fortran module files 55 | *.mod 56 | *.smod 57 | 58 | # Compiled Static libraries 59 | *.lai 60 | *.la 61 | *.a 62 | *.lib 63 | 64 | # Executables 65 | *.exe 66 | *.out 67 | *.app 68 | 69 | ### Automatically added by Hedron's Bazel Compile Commands Extractor: https://github.com/hedronvision/bazel-compile-commands-extractor 70 | # Ignore the `external` link (that is added by `bazel-compile-commands-extractor`). The link differs between macOS/Linux and Windows, so it shouldn't be checked in. The pattern must not end with a trailing `/` because it's a symlink on macOS/Linux. 71 | /external 72 | # Ignore generated output. Although valuable (after all, the primary purpose of `bazel-compile-commands-extractor` is to produce `compile_commands.json`!), it should not be checked in. 73 | /compile_commands.json 74 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | alias( 4 | name = "simpledb", 5 | actual = "//SimpleDB:simpledb", 6 | ) 7 | 8 | alias( 9 | name = "simpledb-test", 10 | actual = "//SimpleDB:simpledb-test", 11 | ) 12 | 13 | alias( 14 | name = "sqlparser", 15 | actual = "//SQLParser:sqlparser", 16 | ) 17 | 18 | alias( 19 | name = "simpledb_service", 20 | actual = "//SimpleDBService:simpledb_service", 21 | ) 22 | 23 | alias( 24 | name = "simpledb_service_py", 25 | actual = "//SimpleDBService:simpledb_service_py" 26 | ) 27 | 28 | alias( 29 | name = "simpledb_server", 30 | actual = "//SimpleDBServer:main", 31 | ) 32 | 33 | alias( 34 | name = "simpledb_client", 35 | actual = "//SimpleDBClient:main", 36 | ) 37 | 38 | test_suite( 39 | name = "test_all", 40 | tests = [ 41 | "//SimpleDBTest:simpledb_test" 42 | ], 43 | ) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Liang Yesheng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SimpleDB 2 | 3 | 清华大学计算机系《数据库系统概论》2022 年大作业项目 DBMS，支持基础 SQL 的解析和执行。 4 | 5 | 所有支持的功能（即支持的 SQL）见[文法文件](SQLParser/Sql.g4)。 6 | 7 |
8 | 9 | 10 |
11 | 12 | ## 架构 13 | 14 | 本项目为 C/S 架构，由 C++ 服务器负责 SQL 的解析和执行，Python 客户端负责处理用户输入，二者之间通过 [gRPC](https://grpc.io) 进行网络传输。 15 | 16 | 各个部分分别为： 17 | 18 | - `SimpleDB`（依赖库）：DBMS，负责数据库管理、SQL 的解析和执行 19 | - `SQLParser`（依赖库）：解析 SQL 并生成 AST 20 | - `SimpleDBService`（依赖库）：gRPC 库，用于支持网络传输 21 | - `SimpleDBServer`（二进制程序）：服务器，通过 `SimpleDB` 处理客户端请求并返回结果 22 | - `SimpleDBClient`（二进制程序）：客户端，处理用户请求，向服务器发送 SQL 语句，并显示返回的结果 23 | - `SimpleDBTest`（单元测试）：对 `SimpleDB` 进行测试 24 | 25 | 各个部分及其使用的第三方库的依赖关系如下： 26 | 27 | ![](docs/dep-graph.svg) 28 | 29 | ## 编译与运行 30 | 31 | > macOS 13.0 和 Ubuntu 22.04 测试编译通过，其余环境未测试。 32 | 33 | ### 环境配置 34 | 35 | 使用 [Bazel](https://bazel.build) 和 clang 编译。 36 | 37 | **对于 macOS**： 38 | 39 | 安装 Xcode command line tools： 40 | 41 | ``` 42 | xcode-select --install 43 | ``` 44 | 45 | 安装 Bazel： 46 | 47 | ``` 48 | brew install bazel 49 | ``` 50 | 51 | **对于 Ubuntu**： 52 | 53 | 安装 Bazel 6.0.0： 54 | 55 | ``` 56 | sudo apt install apt-transport-https curl gnupg -y 57 | curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor >bazel-archive-keyring.gpg 58 | sudo mv bazel-archive-keyring.gpg /usr/share/keyrings 59 | echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list 60 | sudo apt update && sudo apt install bazel-6.0.0 61 | ``` 62 | 63 | 安装 clang 和 lld： 64 | 65 | ``` 66 | sudo apt install clang lld 67 | ``` 68 | 69 | ### 运行 70 | 71 | 运行服务器： 72 | 73 | ``` 74 | bazel run -- :simpledb_server --dir= [--debug | --verbose] [--addr=] 75 | ``` 76 | 77 | 运行交互式客户端： 78 | 79 | ``` 80 | bazel run -- :simpledb_client --server= 81 | ``` 82 | 83 | 从 CSV 中导入数据： 84 | 85 | ``` 86 | bazel run -- :simpledb_client --server= --csv= --db= --table= 87 | ``` 88 | 89 | 运行单元测试： 90 | 91 | ``` 92 | bazel test :test_all 93 | ``` 94 | 95 | 编译所有 target： 96 | 97 | ``` 98 | bazel build //... 99 | ``` 100 | 101 | 编译结束后，运行此命令以生成 `compile_commands.json`： 102 | 103 | ``` 104 | bazel run @hedron_compile_commands//:refresh_all 105 | ``` 106 | 107 | ## 具体实现 108 | 109 | 与具体实现有关的设计，见 [docs/design.md](docs/design.md)。 110 | -------------------------------------------------------------------------------- /SQLParser/.gitignore: -------------------------------------------------------------------------------- 1 | .antlr -------------------------------------------------------------------------------- /SQLParser/BUILD: -------------------------------------------------------------------------------- 1 | load("@rules_antlr//antlr:antlr4.bzl", "antlr") 2 | 3 | antlr( 4 | name = "generated", 5 | srcs = ["Sql.g4"], 6 | package = "SQLParser", 7 | listener = False, 8 | visitor = True, 9 | language = "Cpp", 10 | visibility = ["//visibility:public"], 11 | ) 12 | 13 | # load("@rules_cc//cc:defs.bzl", "cc_library") 14 | 15 | # ANTLR_FLAGS = "-Dlanguage=Cpp -visitor -no-listener -package SQLParser " + \ 16 | # "-Xexact-output-dir -o $(RULEDIR) " 17 | 18 | # genrule( 19 | # name = "antlr-sources", 20 | # srcs = ["Sql.g4"], 21 | # outs = [ 22 | # "SqlLexer.cpp", "SqlParser.cpp", 23 | # "SqlVisitor.cpp", "SqlBaseVisitor.cpp", 24 | # "SqlLexer.h", "SqlParser.h", 25 | # "SqlVisitor.h", "SqlBaseVisitor.h", 26 | # ], 27 | # cmd = "antlr4 " + ANTLR_FLAGS + " $<" 28 | # ) 29 | 30 | cc_library( 31 | name = "sqlparser", 32 | srcs = [":generated"], 33 | deps = [ 34 | ":generated", 35 | "@antlr4_runtimes//:cpp", 36 | ], 37 | includes = ["generated.inc"], 38 | visibility = ["//visibility:public"], 39 | linkstatic = True, 40 | ) 41 | -------------------------------------------------------------------------------- /SQLParser/Sql.g4: -------------------------------------------------------------------------------- 1 | grammar Sql; // Use Sql instead of SQL to avoid name conflict. 2 | 3 | EqualOrAssign: '='; 4 | Less: '<'; 5 | LessEqual: '<='; 6 | Greater: '>'; 7 | GreaterEqual: '>='; 8 | NotEqual: '<>'; 9 | 10 | Count: 'COUNT'; 11 | Average: 'AVG'; 12 | Max: 'MAX'; 13 | Min: 'MIN'; 14 | Sum: 'SUM'; 15 | Null: 'NULL'; 16 | 17 | WhereNot: 'NOT'; 18 | 19 | Identifier: [a-zA-Z_] [a-zA-Z_0-9]*; 20 | Integer: ('-')? [0-9]+; 21 | String: '\'' (~'\'')* '\''; 22 | Float: ('-')? [0-9]+ '.' [0-9]*; 23 | Whitespace: [ \t\n\r]+ -> skip; 24 | Annotation: '-' '-' (~';')+; 25 | 26 | program: statement* EOF; 27 | 28 | statement: 29 | db_statement ';' 30 | | table_statement ';' 31 | | alter_statement ';' 32 | | Annotation ';' 33 | | Null ';'; 34 | 35 | db_statement: 36 | 'CREATE' 'DATABASE' Identifier # create_db 37 | | 'DROP' 'DATABASE' Identifier # drop_db 38 | | 'SHOW' 'DATABASES' # show_dbs 39 | | 'USE' Identifier # use_db 40 | | 'SHOW' 'TABLES' # show_tables 41 | | 'SHOW' 'INDEXES' 'FROM' Identifier # show_indexes; 42 | 43 | table_statement: 44 | 'CREATE' 'TABLE' Identifier '(' field_list ')' # create_table 45 | | 'DROP' 'TABLE' Identifier # drop_table 46 | | 'DESC' Identifier # describe_table 47 | | 'INSERT' 'INTO' Identifier 'VALUES' insert_value_list # insert_into_table 48 | | 'DELETE' 'FROM' Identifier 'WHERE' where_and_clause # delete_from_table 49 | | 'UPDATE' Identifier 'SET' set_clause ( 50 | 'WHERE' where_and_clause 51 | )? # update_table 52 | | select_table # select_table_; 53 | 54 | select_table: 55 | 'SELECT' selectors 'FROM' identifiers ( 56 | 'WHERE' where_and_clause 57 | )? ( 58 | 'LIMIT' Integer ('OFFSET' Integer)? 59 | )?; 60 | 61 | alter_statement: 62 | 'ALTER' 'TABLE' Identifier 'ADD' 'INDEX' '(' identifiers ')' # alter_add_index 63 | | 'ALTER' 'TABLE' Identifier 'DROP' 'INDEX' '(' identifiers ')' # alter_drop_index 64 | | 'ALTER' 'TABLE' Identifier 'DROP' 'PRIMARY' 'KEY' ( 65 | Identifier 66 | )? # alter_table_drop_pk 67 | | 'ALTER' 'TABLE' Identifier 'DROP' 'FOREIGN' 'KEY' '(' Identifier ')' # 68 | alter_table_drop_foreign_key 69 | | 'ALTER' 'TABLE' Identifier 'ADD' 'CONSTRAINT' 'PRIMARY' 'KEY' '(' Identifier ')' # 70 | alter_table_add_pk 71 | | 'ALTER' 'TABLE' Identifier 'ADD' 'CONSTRAINT' 'FOREIGN' 'KEY' '(' Identifier ')' 'REFERENCES' 72 | Identifier '(' Identifier ')' # alter_table_add_foreign_key; 73 | 74 | field_list: field (',' field)*; 75 | 76 | field: 77 | Identifier type_ ('NOT' Null)? ('DEFAULT' value)? # normal_field 78 | | 'PRIMARY' 'KEY' '(' Identifier ')' # primary_key_field 79 | | 'FOREIGN' 'KEY' '(' Identifier ')' 'REFERENCES' Identifier '(' Identifier ')' # 80 | foreign_key_field; 81 | 82 | type_: 'INT' | 'VARCHAR' '(' Integer ')' | 'FLOAT'; 83 | 84 | insert_value: value | 'DEFAULT'; 85 | 86 | insert_value_list: '(' insert_value (',' insert_value)* ')'; 87 | 88 | value_list: '(' value (',' value)* ')'; 89 | 90 | value: Integer | String | Float | Null; 91 | 92 | where_and_clause: where_clause ('AND' where_clause)*; 93 | 94 | where_clause: 95 | column operator_ expression # where_operator_expression 96 | | column 'IS' (WhereNot)? Null # where_null; 97 | 98 | column: (Identifier '.')? Identifier; 99 | 100 | expression: value | column; 101 | 102 | set_clause: 103 | Identifier EqualOrAssign value ( 104 | ',' Identifier EqualOrAssign value 105 | )*; 106 | 107 | selectors: '*' | selector (',' selector)*; 108 | 109 | selector: 110 | column 111 | | aggregator '(' column ')' 112 | | Count '(' '*' ')'; 113 | 114 | identifiers: Identifier (',' Identifier)*; 115 | 116 | operator_: 117 | EqualOrAssign 118 | | Less 119 | | LessEqual 120 | | Greater 121 | | GreaterEqual 122 | | NotEqual; 123 | 124 | aggregator: Count | Average | Max | Min | Sum; 125 | -------------------------------------------------------------------------------- /SimpleDB/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | obj/ -------------------------------------------------------------------------------- /SimpleDB/BUILD: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "simpledb", 3 | srcs = glob(["src/**/*.cc"]), 4 | hdrs = glob(["include/**/*.h"]), 5 | deps = ["//:sqlparser", "//:simpledb_service"], 6 | strip_include_prefix = "include/SimpleDB", 7 | includes = ["include"], 8 | visibility = ["//visibility:public"], 9 | linkstatic = True, 10 | ) 11 | 12 | cc_library( 13 | name = "simpledb-test", 14 | srcs = glob(["src/**/*.cc"]), 15 | hdrs = glob(["include/**/*.h"]), 16 | deps = ["//:sqlparser", "//:simpledb_service"], 17 | strip_include_prefix = "include/SimpleDB", 18 | includes = ["include"], 19 | local_defines = ["DEBUG=1", "TESTING=1"], 20 | visibility = ["//visibility:public"], 21 | linkstatic = True, 22 | ) 23 | -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/DBMS.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_DBMS_H 2 | #define _SIMPLEDB_DBMS_H 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "internal/IndexedTable.h" 12 | #include "internal/ParseTreeVisitor.h" 13 | #include "internal/QueryBuilder.h" 14 | #include "internal/Table.h" 15 | // Keep last... 16 | #include "internal/Service.h" 17 | 18 | /** File structures 19 | * - /system: System tables 20 | * - /databases: Database information 21 | * - /tables: Table information 22 | * - /indexes: Index information 23 | * - /: Database 24 | * - /: Table 25 | * - /index: Index files 26 | * - ///: index for column 27 | */ 28 | 29 | namespace SimpleDB { 30 | class DBMS { 31 | friend class Internal::ParseTreeVisitor; 32 | 33 | public: 34 | DBMS(const std::string &rootPath); 35 | DBMS() = default; 36 | ~DBMS(); 37 | 38 | void init(); 39 | void close(); 40 | 41 | // Execute one or more SQL statement(s). No results will be returned if 42 | // one of the statements has failed even if the effects have taken 43 | // place, for the sheer simplicity. 44 | // @param stream The input stream to read the SQL statement from. 45 | // @throw Error::ExecutionErrorBase 46 | std::vector executeSQL(std::istream &stream); 47 | std::string getCurrentDatabase() const { return currentDatabase; } 48 | #if !TESTING 49 | private: 50 | #endif 51 | std::filesystem::path rootPath; 52 | std::string currentDatabase; 53 | bool initialized = false; 54 | 55 | // === System Tables === 56 | Internal::Table systemDatabaseTable; 57 | Internal::Table systemTablesTable; 58 | Internal::Table systemIndexesTable; 59 | Internal::Table systemForeignKeyTable; 60 | 61 | static std::vector systemDatabaseTableColumns; 62 | static std::vector systemTablesTableColumns; 63 | static std::vector systemIndexesTableColumns; 64 | static std::vector systemForeignKeyTableColumns; 65 | 66 | Internal::ParseTreeVisitor visitor; 67 | std::map openedTables; 68 | 69 | // === Database management methods === 70 | Service::PlainResult createDatabase(const std::string &dbName); 71 | Service::PlainResult dropDatabase(const std::string &dbName); 72 | Service::PlainResult useDatabase(const std::string &dbName); 73 | Service::ShowDatabasesResult showDatabases(); 74 | 75 | Service::ShowTableResult showTables(); 76 | Service::PlainResult createTable( 77 | const std::string &tableName, 78 | const std::vector &columns, 79 | const std::string &primaryKey = std::string(), 80 | const std::vector &foreignKeys = {}); 81 | Service::PlainResult dropTable(const std::string &tableName); 82 | Service::DescribeTableResult describeTable(const std::string &tableName); 83 | 84 | Service::PlainResult alterPrimaryKey(const std::string &tableName, 85 | const std::string &primaryKey, 86 | bool drop); 87 | Service::PlainResult addForeignKey(const std::string &tableName, 88 | const std::string &column, 89 | const std::string &refTable, 90 | const std::string &refColumn); 91 | Service::PlainResult dropForeignKey(const std::string &tableName, 92 | const std::string &column); 93 | Service::PlainResult createIndex(const std::string &tableName, 94 | const std::string &columnName, 95 | bool isPrimaryKey = false); 96 | Service::PlainResult dropIndex(const std::string &tableName, 97 | const std::string &columnName, 98 | bool isPrimaryKey = false); 99 | Service::ShowIndexesResult showIndexes(const std::string &tableName); 100 | 101 | // === CURD methods === 102 | Service::PlainResult insert(const std::string &tableName, 103 | const std::vector &values, 104 | Internal::ColumnBitmap emptyBits); 105 | Service::PlainResult update(Internal::QueryBuilder &builder, 106 | const std::vector &columnNames, 107 | const Internal::Columns &columns); 108 | Service::PlainResult delete_(Internal::QueryBuilder &builder); 109 | Service::QueryResult select(Internal::QueryBuilder &builder); 110 | 111 | Internal::QueryBuilder buildQuery( 112 | const std::vector &tables, 113 | const std::vector &selectors, 114 | const std::vector &valueConditions, 115 | const std::vector &columnConditions, 116 | const std::vector &nullConditions, 117 | int limit, int offset, bool updateOrDelete = false); 118 | 119 | Internal::QueryBuilder buildQueryForUpdateOrDelete( 120 | const std::string &table, 121 | const std::vector &valueConditions, 122 | const std::vector &columnConditions, 123 | const std::vector &nullConditions); 124 | 125 | // === System tables === 126 | void initSystemTable(Internal::Table *table, const std::string &name, 127 | const std::vector columns); 128 | 129 | // === Helper methods === 130 | void checkUseDatabase(); 131 | void clearCurrentDatabase(); 132 | 133 | std::filesystem::path getSystemTablePath(const std::string &name); 134 | std::filesystem::path getUserTablePath(const std::string database, 135 | const std::string &name); 136 | std::filesystem::path getIndexPath(const std::string &database, 137 | const std::string &table, 138 | const std::string &column); 139 | 140 | Service::PlainResult makePlainResult(const std::string &msg, 141 | int affectedRows = -1); 142 | 143 | struct ForeignKeyInfo { 144 | Internal::RecordID rid; 145 | std::string database; 146 | std::string table; 147 | std::string column; 148 | std::string refTable; 149 | std::string refColumn; 150 | }; 151 | std::pair findDatabase( 152 | const std::string &dbName); 153 | std::pair findTable( 154 | const std::string &database, const std::string &tableName); 155 | Internal::QueryBuilder::Result findAllTables(const std::string &database); 156 | std::pair findIndex( 157 | const std::string &database, const std::string &table, 158 | const std::string &columnName); 159 | Internal::QueryBuilder::Result findIndexes(const std::string &database, 160 | const std::string &table); 161 | std::vector findForeignKeys(const std::string &database, 162 | const std::string &table, 163 | const std::string &column, 164 | const std::string &refTable, 165 | const std::string &refColumn); 166 | bool hasReferencingRecord( 167 | const Internal::Table *oriTable, 168 | const std::vector &referencingColumns, 169 | const Internal::Columns &allNewColumns); 170 | std::pair getTable( 171 | const std::string &tableName); 172 | std::pair> getIndex( 173 | const std::string &database, const std::string &table, 174 | const std::string &column); 175 | std::shared_ptr newIndexedTable( 176 | Internal::Table *table); 177 | }; 178 | 179 | }; // namespace SimpleDB 180 | 181 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/Error.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_BASE_ERROR_H 2 | #define _SIMPLEDB_BASE_ERROR_H 3 | 4 | #include 5 | #include 6 | 7 | namespace SimpleDB { 8 | 9 | struct BaseError : std::exception { 10 | BaseError(std::string description) : description(description) {} 11 | virtual const char* what() const noexcept { return description.c_str(); } 12 | 13 | private: 14 | std::string description; 15 | }; 16 | 17 | #define DECLARE_ERROR_CLASS(ErrorClass, BaseClass, description) \ 18 | struct ErrorClass##ErrorBase : BaseError { \ 19 | ErrorClass##ErrorBase(std::string _description) \ 20 | : BaseError(_description) {} \ 21 | ErrorClass##ErrorBase() : BaseError(description) {} \ 22 | }; 23 | 24 | #define DECLARE_ERROR(ErrorType, BaseClass, description) \ 25 | struct ErrorType##Error : BaseClass { \ 26 | ErrorType##Error(std::string _description) \ 27 | : BaseClass(std::string(description) + ": " + _description) {} \ 28 | ErrorType##Error() : BaseClass(description) {} \ 29 | }; 30 | 31 | // Internal errors. 32 | namespace Internal { 33 | 34 | DECLARE_ERROR_CLASS(Internal, BaseError, "Internal error"); 35 | 36 | // ==== I/O Error ==== 37 | DECLARE_ERROR_CLASS(IO, InternalErrorBase, "I/O error"); 38 | 39 | DECLARE_ERROR(OpenFile, IOErrorBase, "Fail to open file"); 40 | DECLARE_ERROR(CreateFile, IOErrorBase, "Fail to create file"); 41 | DECLARE_ERROR(CloseFile, IOErrorBase, "Fail to close file"); 42 | DECLARE_ERROR(ReadFile, IOErrorBase, "Fail to read file"); 43 | DECLARE_ERROR(WriteFile, IOErrorBase, "Fail to write file"); 44 | DECLARE_ERROR(DeleteFile, IOErrorBase, "Fail to delete file"); 45 | DECLARE_ERROR(FileExists, IOErrorBase, "File already exists"); 46 | DECLARE_ERROR(InvalidDescriptor, IOErrorBase, "Invalid file descriptor"); 47 | DECLARE_ERROR(InvalidPageNumber, IOErrorBase, "Invalid file descriptor"); 48 | DECLARE_ERROR(OpenFileExceeded, IOErrorBase, 49 | "Number of opened files has exceeded"); 50 | DECLARE_ERROR(InvalidPageHandle, IOErrorBase, "Invalid page handle"); 51 | 52 | // ==== Table Operation Error ==== 53 | DECLARE_ERROR_CLASS(Table, InternalErrorBase, "Table operation error"); 54 | 55 | DECLARE_ERROR(ReadTable, TableErrorBase, "Fail to read table"); 56 | DECLARE_ERROR(CreateTable, TableErrorBase, "Fail to create table"); 57 | DECLARE_ERROR(DuplicateColumnName, TableErrorBase, 58 | "Duplicate column name found"); 59 | DECLARE_ERROR(TableNotInitialized, TableErrorBase, 60 | "The table is not initialized before used"); 61 | DECLARE_ERROR(ColumnSerialization, TableErrorBase, 62 | "Fail to serialize column from byte stream"); 63 | DECLARE_ERROR(InvalidSlot, TableErrorBase, "Invalid page/slot number"); 64 | DECLARE_ERROR(InvalidColumnSize, TableErrorBase, "Invalid column size"); 65 | DECLARE_ERROR(InvalidColumnIndex, TableErrorBase, "Invalid column index"); 66 | DECLARE_ERROR(ColumnFull, TableErrorBase, "The column is full"); 67 | DECLARE_ERROR(ColumnExists, TableErrorBase, "The column already exists"); 68 | DECLARE_ERROR(InvalidPageMeta, TableErrorBase, "The page meta is invalid"); 69 | DECLARE_ERROR(TooManyColumns, TableErrorBase, "Too many columns"); 70 | DECLARE_ERROR(TooManyForeignKeys, TableErrorBase, "Too many foreign keys"); 71 | DECLARE_ERROR(InvalidPrimaryKey, TableErrorBase, "Invalid primary key"); 72 | DECLARE_ERROR(InvalidForeignKey, TableErrorBase, "Invalid foreign key"); 73 | DECLARE_ERROR(PrimaryKeyExists, TableErrorBase, "Primary key exists"); 74 | DECLARE_ERROR(PrimaryKeyNotExists, TableErrorBase, 75 | "Primary key does not exist"); 76 | DECLARE_ERROR(NullValueFoundInNotNullColumn, TableErrorBase, 77 | "Null value found in not null column"); 78 | DECLARE_ERROR(NullValueGivenForNotNullColumn, TableErrorBase, 79 | "Null value given for not null column"); 80 | DECLARE_ERROR(ValueNotGiven, TableErrorBase, 81 | "The value of a column without default value is not given"); 82 | DECLARE_ERROR(IncorrectColumnNum, TableErrorBase, 83 | "Incorrect number of columns are given"); 84 | DECLARE_ERROR(ForeignKeyViolation, TableErrorBase, 85 | "Violating foreign key constraints"); 86 | 87 | // ==== Iterator Error ==== 88 | DECLARE_ERROR_CLASS(Iterator, InternalErrorBase, "Iterator error"); 89 | 90 | DECLARE_ERROR(InvalidColumnName, IteratorErrorBase, "Invalid column name"); 91 | DECLARE_ERROR(UnexpedtedOperator, IteratorErrorBase, "Unexpected operator"); 92 | DECLARE_ERROR(InvalidOperator, IteratorErrorBase, "Invalid operator"); 93 | DECLARE_ERROR(InvalidRegex, IteratorErrorBase, 94 | "Invalid input regular expression"); 95 | 96 | // ==== Index Error ==== 97 | DECLARE_ERROR_CLASS(Index, InternalErrorBase, "Index error"); 98 | 99 | DECLARE_ERROR(InvalidIndexMeta, IndexErrorBase, "Invalid index meta"); 100 | DECLARE_ERROR(ReadIndex, IndexErrorBase, "Fail to read index"); 101 | DECLARE_ERROR(CreateIndex, IndexErrorBase, "Fail to create index"); 102 | DECLARE_ERROR(InvalidIndexType, IndexErrorBase, "Invalid index column type"); 103 | DECLARE_ERROR(IndexNotInitialized, IndexErrorBase, 104 | "The index is not initialized yet"); 105 | DECLARE_ERROR(IndexKeyExists, IndexErrorBase, "Duplicate index key found"); 106 | DECLARE_ERROR(IndexKeyNotExists, IndexErrorBase, "The index does not exist"); 107 | DECLARE_ERROR(WriteOnReadOnlyIndex, IndexErrorBase, 108 | "Internal: trying to write on a read-only index"); 109 | 110 | // ==== QueryBuilder Error ==== 111 | DECLARE_ERROR_CLASS(QueryBuilder, InternalErrorBase, "QueryBuilder error"); 112 | DECLARE_ERROR(MultipleScan, QueryBuilderErrorBase, 113 | "Multiple tables are provided for scan"); 114 | DECLARE_ERROR(InvalidLimit, QueryBuilderErrorBase, "Invalid limit given"); 115 | DECLARE_ERROR(NoScanDataSource, QueryBuilderErrorBase, 116 | "No scan data source provided"); 117 | DECLARE_ERROR(ColumnNotFound, QueryBuilderErrorBase, 118 | "Column not found in table"); 119 | DECLARE_ERROR(Aggregator, QueryBuilderErrorBase, "Invalid aggregator"); 120 | DECLARE_ERROR(AmbiguousColumn, QueryBuilderErrorBase, "Ambiguous column"); 121 | } // namespace Internal 122 | 123 | // Exposed errors. 124 | namespace Error { 125 | 126 | DECLARE_ERROR_CLASS(Execution, BaseError, "Fail to execute SQL"); 127 | DECLARE_ERROR(Syntax, ExecutionErrorBase, "Syntax error"); 128 | DECLARE_ERROR(Unknown, ExecutionErrorBase, "Unknown exception"); 129 | DECLARE_ERROR(Internal, ExecutionErrorBase, "Internal error"); 130 | DECLARE_ERROR(IncompatableValue, ExecutionErrorBase, 131 | "Incompatable value error"); 132 | DECLARE_ERROR(Uninitialized, ExecutionErrorBase, "DMBS is uninitialized"); 133 | DECLARE_ERROR(Initialization, ExecutionErrorBase, "Fail to initialize DMBS"); 134 | DECLARE_ERROR(InvalidDatabaseName, ExecutionErrorBase, "Invalid database name"); 135 | DECLARE_ERROR(DatabaseExists, ExecutionErrorBase, 136 | "The database already exists"); 137 | DECLARE_ERROR(CreateDatabase, ExecutionErrorBase, "Fail to create database"); 138 | DECLARE_ERROR(DatabaseNotExist, ExecutionErrorBase, 139 | "The database does not exists"); 140 | DECLARE_ERROR(DatabaseNotSelected, ExecutionErrorBase, 141 | "No database is selected"); 142 | DECLARE_ERROR(TableExists, ExecutionErrorBase, "The table already exists"); 143 | DECLARE_ERROR(InvalidTableName, ExecutionErrorBase, "Invalid table name"); 144 | DECLARE_ERROR(TableNotExists, ExecutionErrorBase, "The table does not exist"); 145 | DECLARE_ERROR(MultiplePrimaryKey, ExecutionErrorBase, 146 | "More than one primary key is given"); 147 | DECLARE_ERROR(CreateTable, ExecutionErrorBase, "Fail to create table"); 148 | DECLARE_ERROR(DropTable, ExecutionErrorBase, "Fail to drop table"); 149 | DECLARE_ERROR(AlterPrimaryKey, ExecutionErrorBase, "Fail to alter primary key"); 150 | DECLARE_ERROR(AlterForeignKey, ExecutionErrorBase, "Fail to alter foreign key"); 151 | DECLARE_ERROR(AlterIndex, ExecutionErrorBase, "Fail to alter index"); 152 | DECLARE_ERROR(Insert, ExecutionErrorBase, "INSERT statement failed"); 153 | DECLARE_ERROR(Select, ExecutionErrorBase, "SELECT statement failed"); 154 | DECLARE_ERROR(Update, ExecutionErrorBase, "UPDATE statement failed"); 155 | DECLARE_ERROR(Delete, ExecutionErrorBase, "DELETE statement failed"); 156 | 157 | } // namespace Error 158 | 159 | #undef DECLARE_ERROR 160 | #undef DECLARE_ERROR_CLASS 161 | 162 | } // namespace SimpleDB 163 | 164 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/SimpleDB.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_SIMPLEDB_H 2 | #define _SIMPLEDB_SIMPLEDB_H 3 | 4 | #include "Error.h" 5 | #include "internal/Logger.h" 6 | 7 | // The latter is just for code highliging. 8 | #if TESTING 9 | #include "internal/Index.h" 10 | #include "internal/PageFile.h" 11 | #endif 12 | 13 | // Include at last, as it may include other headers that conflict with existing 14 | // declarations. 15 | #include "DBMS.h" 16 | 17 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/CacheManager.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_CACHE_MANAGER_H 2 | #define _SIMPLEDB_CACHE_MANAGER_H 3 | 4 | #include 5 | 6 | #include "Error.h" 7 | #include "internal/FileManager.h" 8 | #include "internal/LinkedList.h" 9 | 10 | namespace SimpleDB { 11 | namespace Internal { 12 | 13 | // Forward declaration. 14 | struct PageHandle; 15 | 16 | class CacheManager { 17 | friend struct PageHandle; 18 | 19 | public: 20 | CacheManager(FileManager *fileManager); 21 | ~CacheManager(); 22 | 23 | // Load a page from the cache (or the disk). 24 | PageHandle getHandle(FileDescriptor fd, int page); 25 | 26 | // Renew the page handle, if it's invalidated. 27 | PageHandle renew(const PageHandle &handle); 28 | 29 | // Get the pointer to the buffer, NULL if the handle is invalid (i.e. 30 | // outdated), which indicates that the caller should re-load the page using 31 | // renew(). The result can be cached between two cache operations for 32 | // better performance and convenience. 33 | char *load(const PageHandle &handle); 34 | 35 | // The unsafe version of `load`, in which the validity of the handle is not 36 | // examined. 37 | char *loadRaw(const PageHandle &handle); 38 | 39 | // Mark the page as dirty, should be called after every write to the buffer. 40 | // The handle must be validated via validate() before calling this function, 41 | // otherwise InvalidPageHandleError might be thrown. 42 | void markDirty(const PageHandle &handle); 43 | 44 | // Write the cache back to the disk if it is dirty, and remove the cache. If 45 | // the cache does not exist, do nothing. The handle must be validated via 46 | // validate() before calling this function, otherwise InvalidPageHandleError 47 | // might be thrown. 48 | void writeBack(const PageHandle &handle); 49 | 50 | // A handler to do some cleanup before the file manager closes the file. 51 | void onCloseFile(FileDescriptor fd) noexcept(false); 52 | 53 | // Write back all pages and destroy the cache manager. 54 | void close(); 55 | 56 | #if TESTING 57 | // ==== Testing-only methods ==== 58 | void discard(FileDescriptor fd, int page); 59 | void discardAll(FileDescriptor fd); 60 | void discardAll(); 61 | #endif 62 | 63 | #if !TESTING 64 | private: 65 | #endif 66 | FileManager *fileManager; 67 | 68 | struct PageMeta { 69 | // The associated file descriptor. 70 | FileDescriptor fd; 71 | // The page number. 72 | int page; 73 | 74 | PageMeta() : fd(-1), page(-1) {} 75 | PageMeta(FileDescriptor fd, int page) : fd(fd), page(page) {} 76 | }; 77 | 78 | struct PageCache { 79 | PageMeta meta; 80 | int id; // The index in the buffer. 81 | bool dirty; 82 | char buf[PAGE_SIZE]; 83 | 84 | int generation = 0; 85 | 86 | // A reverse pointer to the node in the linked list. 87 | LinkedList::Node *nodeInActiveCache = nullptr; 88 | 89 | // Replace this cache with another page. 90 | void reset(const PageMeta &meta) { 91 | this->meta = meta; 92 | dirty = false; 93 | nodeInActiveCache = nullptr; 94 | // We don't need to bump the generation number here. It's done 95 | // during write back. 96 | } 97 | }; 98 | 99 | std::vector> activeCacheMapVec; 100 | 101 | LinkedList freeCache; 102 | LinkedList activeCache; 103 | 104 | PageCache *cacheBuf; 105 | bool closed = false; 106 | 107 | // Write the cache back to the disk if it is dirty, and remove the cache. 108 | void writeBack(PageCache *cache); 109 | 110 | // Get the cache for certain page. Claim a slot (and load from disk) if it 111 | // is not cached. 112 | PageCache *getPageCache(FileDescriptor fd, int page) noexcept(false); 113 | }; 114 | 115 | } // namespace Internal 116 | } // namespace SimpleDB 117 | 118 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Column.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_COLUMN_H 2 | #define _SIMPLEDB_COLUMN_H 3 | 4 | #include 5 | #include 6 | 7 | #include "internal/Macros.h" 8 | 9 | namespace SimpleDB { 10 | namespace Internal { 11 | 12 | using ColumnSizeType = uint32_t; 13 | 14 | enum DataType { 15 | INT, // 32 bit integer 16 | FLOAT, // 32 bit float 17 | VARCHAR // length-variable string 18 | }; 19 | 20 | union ColumnValue { 21 | int intValue; 22 | float floatValue; 23 | char stringValue[MAX_COLUMN_SIZE]; 24 | }; 25 | 26 | struct Column { 27 | DataType type; 28 | ColumnSizeType size; 29 | bool isNull = false; 30 | ColumnValue data; 31 | 32 | // Initialize a null column. 33 | static Column nullColumn(DataType type, ColumnSizeType size); 34 | static Column nullIntColumn(); 35 | static Column nullFloatColumn(); 36 | static Column nullVarcharColumn(ColumnSizeType size); 37 | 38 | // Convenienve constructors. 39 | Column(int data); 40 | Column(float data); 41 | Column(const char *data, int maxLength); 42 | 43 | Column() = default; 44 | }; 45 | 46 | using Columns = std::vector; 47 | using ColumnBitmap = int16_t; 48 | 49 | static_assert(sizeof(ColumnBitmap) == sizeof(COLUMN_BITMAP_ALL)); 50 | 51 | struct ColumnInfo { 52 | std::string tableName; 53 | std::string columnName; 54 | DataType type; 55 | 56 | bool operator==(const ColumnInfo &rhs) { 57 | return tableName == rhs.tableName && columnName == rhs.columnName; 58 | } 59 | 60 | std::string getDesc() const { 61 | return tableName.empty() ? columnName : tableName + "." + columnName; 62 | } 63 | }; 64 | 65 | struct RecordID { 66 | int page; 67 | int slot; 68 | 69 | bool operator==(const RecordID &rhs) const; 70 | bool operator!=(const RecordID &rhs) const; 71 | bool operator>(const RecordID &rhs) const; 72 | static const RecordID NULL_RECORD; 73 | }; 74 | 75 | } // namespace Internal 76 | } // namespace SimpleDB 77 | 78 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Comparer.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_COMPARER_H 2 | #define _SIMPLEDB_COMPARER_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "internal/Macros.h" 9 | 10 | namespace SimpleDB { 11 | namespace Internal { 12 | 13 | // === Internal classes only for comparision === 14 | class _String { 15 | public: 16 | _String(const char *data) : str(data) {} 17 | bool operator==(const _String &rhs) const { 18 | return strcmp(str, rhs.str) == 0; 19 | } 20 | bool operator!=(const _String &rhs) const { 21 | return strcmp(str, rhs.str) != 0; 22 | } 23 | bool operator<(const _String &rhs) const { 24 | return strcmp(str, rhs.str) < 0; 25 | } 26 | bool operator<=(const _String &rhs) const { 27 | return strcmp(str, rhs.str) <= 0; 28 | } 29 | bool operator>(const _String &rhs) const { 30 | return strcmp(str, rhs.str) > 0; 31 | } 32 | bool operator>=(const _String &rhs) const { 33 | return strcmp(str, rhs.str) >= 0; 34 | } 35 | 36 | private: 37 | const char *str; 38 | }; 39 | 40 | class _Int { 41 | public: 42 | _Int(const char *data) : value(*(int *)data) {} 43 | bool operator==(const _Int &rhs) const { return value == rhs.value; } 44 | bool operator!=(const _Int &rhs) const { return value != rhs.value; } 45 | bool operator<(const _Int &rhs) const { return value < rhs.value; } 46 | bool operator<=(const _Int &rhs) const { return value <= rhs.value; } 47 | bool operator>(const _Int &rhs) const { return value > rhs.value; } 48 | bool operator>=(const _Int &rhs) const { return value >= rhs.value; } 49 | 50 | private: 51 | int value; 52 | }; 53 | 54 | class _Float { 55 | public: 56 | _Float(const char *data) : value(*(float *)data) {} 57 | bool operator==(const _Float &rhs) const { 58 | return std::fabs(value - rhs.value) <= EQUAL_PRECISION; 59 | } 60 | bool operator!=(const _Float &rhs) const { 61 | return std::fabs(value - rhs.value) <= EQUAL_PRECISION; 62 | } 63 | bool operator<(const _Float &rhs) const { return value < rhs.value; } 64 | bool operator<=(const _Float &rhs) const { 65 | return *this < rhs || *this == rhs; 66 | } 67 | bool operator>(const _Float &rhs) const { return value > rhs.value; } 68 | bool operator>=(const _Float &rhs) const { 69 | return *this > rhs || *this == rhs; 70 | } 71 | 72 | private: 73 | float value; 74 | }; 75 | 76 | } // namespace Internal 77 | 78 | } // namespace SimpleDB 79 | 80 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/FileCoordinator.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_FILE_COORDINATOR_H 2 | #define _SIMPLEDB_FILE_COORDINATOR_H 3 | 4 | #include "internal/CacheManager.h" 5 | #include "internal/FileDescriptor.h" 6 | #include "internal/FileManager.h" 7 | #include "internal/PageHandle.h" 8 | 9 | namespace SimpleDB { 10 | namespace Internal { 11 | 12 | // A proxy class to coordinate page access in FileManager and CacheManger, which 13 | // should be the only interface to access the storage. It is not responsible for 14 | // error handling. 15 | class FileCoordinator { 16 | public: 17 | ~FileCoordinator(); 18 | 19 | static FileCoordinator shared; 20 | 21 | void createFile(const std::string &fileName); 22 | FileDescriptor openFile(const std::string &fileName); 23 | void closeFile(FileDescriptor fd); 24 | void deleteFile(const std::string &fileName); 25 | PageHandle getHandle(FileDescriptor fd, int page); 26 | // A safe method to load a page with a handle (valid/invalid). Note that the 27 | // handle might be renewed. 28 | char *load(PageHandle *handle); 29 | // Directly return the buffer without validation. 30 | inline char *loadRaw(const PageHandle &handle) { 31 | return cacheManager->loadRaw(handle); 32 | } 33 | void markDirty(const PageHandle &handle); 34 | PageHandle renew(const PageHandle &handle); 35 | 36 | #if !TESTING 37 | private: 38 | #endif 39 | FileCoordinator(); 40 | 41 | FileManager *fileManager; 42 | CacheManager *cacheManager; 43 | }; 44 | 45 | } // namespace Internal 46 | } // namespace SimpleDB 47 | 48 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/FileDescriptor.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_FILE_DESCRIPTOR 2 | #define _SIMPLEDB_FILE_DESCRIPTOR 3 | 4 | namespace SimpleDB { 5 | namespace Internal { 6 | 7 | // The type for an external, transparent file descriptor. It's actually an 8 | // integer, but we wrap it for type checking. 9 | struct FileDescriptor { 10 | operator int() { return value; } 11 | explicit FileDescriptor(const int value) : value(value) {} 12 | FileDescriptor() : value(-1) {} 13 | 14 | int value; 15 | }; 16 | 17 | bool inline operator==(const FileDescriptor &lhs, const FileDescriptor &rhs) { 18 | return lhs.value == rhs.value; 19 | } 20 | 21 | } // namespace Internal 22 | } // namespace SimpleDB 23 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/FileManager.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_FILE_MANAGER_H 2 | #define _SIMPLEDB_FILE_MANAGER_H 3 | 4 | #include 5 | #include 6 | 7 | #include "Error.h" 8 | #include "internal/FileDescriptor.h" 9 | #include "internal/Macros.h" 10 | 11 | namespace SimpleDB { 12 | namespace Internal { 13 | 14 | class FileManager { 15 | public: 16 | // Error of file operations. 17 | 18 | // The maximum number of files that can be opened at the same time. 19 | static const int MAX_OPEN_FILES = 64; 20 | 21 | void createFile(const std::string &fileName) noexcept(false); 22 | FileDescriptor openFile(const std::string &fileName) noexcept(false); 23 | void closeFile(FileDescriptor fd) noexcept(false); 24 | void deleteFile(const std::string &fileName) noexcept(false); 25 | void readPage(FileDescriptor fd, int page, char *data, 26 | bool couldFail = false) noexcept(false); 27 | void writePage(FileDescriptor fd, int page, char *data) noexcept(false); 28 | 29 | // Check if the file descriptor is valid. 30 | bool validate(FileDescriptor fd); 31 | 32 | #if !TESTING 33 | private: 34 | #endif 35 | struct OpenedFile { 36 | std::string fileName; 37 | FILE *fd = nullptr; 38 | }; 39 | OpenedFile openedFiles[MAX_OPEN_FILES]; 40 | uint64_t descriptorBitmap = 0; 41 | 42 | FileDescriptor genNewDescriptor(FILE *fd, const std::string &fileName); 43 | }; 44 | 45 | } // namespace Internal 46 | } // namespace SimpleDB 47 | 48 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Index.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_INDEX_H 2 | #define _SIMPLEDB_INDEX_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "internal/PageFile.h" 11 | #include "internal/Table.h" 12 | 13 | namespace SimpleDB { 14 | namespace Internal { 15 | 16 | class Index { 17 | private: 18 | using NodeIndex = int; 19 | using IterateFunc = std::function; 20 | 21 | public: 22 | Index() = default; 23 | ~Index(); 24 | void open(const std::string &file); 25 | void create(const std::string &file); 26 | void close(); 27 | 28 | void insert(int key, bool isNull, RecordID id); 29 | void remove(int key, bool isNull, RecordID rid); 30 | 31 | using Range = std::pair; // [first, second] 32 | 33 | bool has(int key, bool isNull); 34 | void iterateEq(int key, bool isNull, IterateFunc func); 35 | void iterateRange(Range, IterateFunc func); 36 | std::vector findEq(int key, bool isNull); 37 | void setReadOnly(); 38 | 39 | #ifndef TESTING 40 | private: 41 | #endif 42 | struct IndexMeta { 43 | uint16_t headCanary = INDEX_META_CANARY; 44 | int numNode; 45 | int numEntry; 46 | int firstFreeSlot; 47 | NodeIndex rootNode; 48 | uint16_t tailCanary = INDEX_META_CANARY; 49 | }; 50 | 51 | struct IndexEntry { 52 | int key; 53 | bool isNull = false; 54 | RecordID record; 55 | 56 | bool operator==(const IndexEntry &rhs) const; 57 | bool operator>(const IndexEntry &rhs) const; 58 | }; 59 | 60 | struct SharedNode { 61 | bool isLeaf; 62 | NodeIndex index; 63 | IndexEntry entry[MAX_NUM_ENTRY_PER_NODE + 1]; 64 | int numEntry; 65 | int parent; 66 | }; 67 | 68 | struct LeafNode { 69 | /* Keep first */ 70 | SharedNode shared; 71 | 72 | int32_t validBitmap; 73 | NodeIndex next; 74 | NodeIndex previous; 75 | 76 | inline bool valid(int index) { 77 | return (validBitmap & (1L << index)) != 0; 78 | } 79 | 80 | static_assert(sizeof(decltype(validBitmap)) * 8 >= 81 | MAX_NUM_ENTRY_PER_NODE); 82 | 83 | // TODO: Pointer to the next leaf node 84 | }; 85 | 86 | struct InnerNode { 87 | static const int NULL_NODE = -1; 88 | 89 | /* Keep first */ 90 | SharedNode shared; 91 | 92 | int children[MAX_NUM_CHILD_PER_NODE + 1]; 93 | int numChildren; 94 | }; 95 | 96 | static const NodeIndex NULL_NODE_INDEX = -1; 97 | 98 | static_assert(sizeof(IndexMeta) <= PAGE_SIZE); 99 | static_assert(sizeof(LeafNode) <= INDEX_SLOT_SIZE); 100 | static_assert(sizeof(InnerNode) <= INDEX_SLOT_SIZE); 101 | 102 | FileDescriptor fd; 103 | IndexMeta meta; 104 | 105 | bool initialized = false; 106 | bool readOnly = false; 107 | 108 | void flushMeta(); 109 | void checkInit() noexcept(false); 110 | 111 | // === Internal helper methods === 112 | 113 | std::tuple findEntry(const IndexEntry &entry, 114 | bool skipInvalid); 115 | // std::tuple findLeftmostLeafNode(int key); 116 | NodeIndex createNewLeafNode(NodeIndex parent); 117 | NodeIndex createNewInnerNode(NodeIndex parent); 118 | SharedNode *getNewNode(NodeIndex parent); 119 | 120 | // Insert the entry WITHOUT checking the constraints. 121 | int insertEntry(SharedNode *sharedNode, const IndexEntry &entry); 122 | void checkOverflowFrom(NodeIndex index); 123 | inline PageHandle getHandle(NodeIndex index) { 124 | int page = index / NUM_INDEX_SLOT; 125 | return PF::getHandle(fd, page + 1); 126 | } 127 | template 128 | T load(NodeIndex index, PageHandle handle) { 129 | char *ptr = PF::loadRaw(handle); 130 | ptr += (index % NUM_INDEX_SLOT) * INDEX_SLOT_SIZE; 131 | return (T)ptr; 132 | } 133 | 134 | #if DEBUG 135 | void dump(); 136 | #endif 137 | }; 138 | 139 | } // namespace Internal 140 | } // namespace SimpleDB 141 | 142 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/IndexedTable.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_INDEXED_TABLE_H 2 | #define _SIMPLEDB_INDEXED_TABLE_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "internal/Index.h" 9 | #include "internal/QueryDataSource.h" 10 | #include "internal/QueryFilter.h" 11 | #include "internal/Table.h" 12 | 13 | namespace SimpleDB { 14 | namespace Internal { 15 | 16 | class IndexedTable : public QueryDataSource { 17 | public: 18 | using GetIndexFunc = std::function( 19 | const std::string &table, const std::string &column)>; 20 | IndexedTable(Table *table, GetIndexFunc getIndex); 21 | 22 | virtual void iterate(IterateCallback callback) override; 23 | virtual std::vector getColumnInfo() override; 24 | virtual bool acceptCondition( 25 | const CompareValueCondition &condition) override; 26 | 27 | Table *getTable(); 28 | 29 | #if !TESTING 30 | private: 31 | #endif 32 | Table *table; 33 | GetIndexFunc getIndex; 34 | std::shared_ptr index; 35 | std::string columnName; 36 | std::vector ranges; 37 | bool emptySet = false; 38 | 39 | Index::Range makeRange(const CompareValueCondition &condition); 40 | void collapseRanges(); 41 | }; 42 | 43 | } // namespace Internal 44 | } // namespace SimpleDB 45 | 46 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/JoinedTable.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_JOINED_TABLE_H 2 | #define _SIMPLEDB_JOINED_TABLE_H 3 | 4 | #include 5 | #include 6 | 7 | #include "internal/IndexedTable.h" 8 | #include "internal/QueryDataSource.h" 9 | 10 | namespace SimpleDB { 11 | namespace Internal { 12 | 13 | class JoinedTable : public QueryDataSource { 14 | public: 15 | JoinedTable() = default; 16 | void append(std::shared_ptr table); 17 | void close(); 18 | 19 | virtual void iterate(IterateCallback callback) override; 20 | virtual std::vector getColumnInfo() override; 21 | virtual bool acceptCondition( 22 | const CompareValueCondition &condition) override; 23 | 24 | private: 25 | std::vector> tables; 26 | }; 27 | 28 | } // namespace Internal 29 | } // namespace SimpleDB 30 | 31 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/LinkedList.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_LINKED_LIST_H 2 | #define _SIMPLEDB_LINKED_LIST_H 3 | 4 | namespace SimpleDB { 5 | namespace Internal { 6 | 7 | template 8 | class LinkedList { 9 | public: 10 | LinkedList() { 11 | head = new Node(); 12 | tail = head; 13 | head->next = nullptr; 14 | head->prev = nullptr; 15 | } 16 | 17 | struct Node { 18 | T *data; 19 | Node *next; 20 | Node *prev; 21 | }; 22 | 23 | /** 24 | * +-------+ +-------------+ 25 | * | guard | -> | actual tail | -> ... 26 | * +-------+ +-------------+ 27 | */ 28 | Node *insertHead(T *data) { 29 | Node *node = new Node(); 30 | node->data = data; 31 | head->next = node; 32 | node->next = nullptr; 33 | node->prev = head; 34 | head = node; 35 | _size++; 36 | return node; 37 | } 38 | 39 | T *removeTail() { 40 | if (head == tail) { 41 | return nullptr; 42 | } 43 | return remove(tail->next); 44 | } 45 | 46 | T *last() { 47 | if (head == tail) { 48 | return nullptr; 49 | } 50 | return tail->next->data; 51 | } 52 | 53 | T *remove(Node *node) { 54 | node->prev->next = node->next; 55 | if (node != head) { 56 | node->next->prev = node->prev; 57 | } else { 58 | head = node->prev; 59 | } 60 | 61 | T *data = node->data; 62 | delete node; 63 | _size--; 64 | 65 | return data; 66 | } 67 | 68 | inline int size() { return _size; }; 69 | 70 | #if !TESTING 71 | private: 72 | #endif 73 | Node *head; 74 | Node *tail; 75 | int _size = 0; 76 | }; 77 | 78 | } // namespace Internal 79 | } // namespace SimpleDB 80 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Logger.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_LOGGER_H 2 | #define _SIMPLEDB_LOGGER_H 3 | 4 | #include 5 | 6 | namespace SimpleDB { 7 | namespace Internal { 8 | 9 | enum LogLevel { VERBOSE, DEBUG_, NOTICE, WARNING, ERROR, SILENT }; 10 | 11 | static const char* logLevelNames[] = {"VERBOSE", "DEBUG", "NOTICE", 12 | "WARNING", "ERROR", "SILENT"}; 13 | 14 | class Logger { 15 | public: 16 | static void setErrorStream(FILE* stream) { errorStream = stream; } 17 | static void setLogLevel(LogLevel level) { displayMinLevel = level; } 18 | static LogLevel getLogLevel() { return displayMinLevel; } 19 | static void log(LogLevel level, const char* fmt, ...) 20 | #ifdef __GNUC__ 21 | __attribute__((format(printf, 2, 3))); 22 | #else 23 | ; 24 | #endif 25 | 26 | private: 27 | static LogLevel displayMinLevel; 28 | static FILE* errorStream; 29 | }; 30 | 31 | } // namespace Internal 32 | } // namespace SimpleDB 33 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Macros.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_MACROS_H 2 | #define _SIMPLEDB_MACROS_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | namespace SimpleDB { 9 | namespace Internal { 10 | 11 | const int PAGE_SIZE = 8192; 12 | const int NUM_BUFFER_PAGE = 1024; 13 | 14 | const int MAX_VARCHAR_LEN = 256 - 1; 15 | const int MAX_COLUMN_SIZE = MAX_VARCHAR_LEN + 1; 16 | const int MAX_COLUMNS = 16; 17 | const int MAX_COLUMN_NAME_LEN = 64; 18 | const int MAX_TABLE_NAME_LEN = 64; 19 | const int MAX_DATABASE_NAME_LEN = MAX_VARCHAR_LEN; 20 | const int MAX_FOREIGN_KEYS = 12; 21 | 22 | const int MAX_SLOT_PER_PAGE = 64; 23 | const int16_t COLUMN_BITMAP_ALL = ~0; 24 | static_assert(MAX_SLOT_PER_PAGE < NUM_BUFFER_PAGE); 25 | 26 | const uint16_t TABLE_META_CANARY = 0xDDBB; 27 | const uint16_t PAGE_META_CANARY = 0xDBDB; 28 | const uint16_t INDEX_META_CANARY = 0xDADA; 29 | const uint16_t EMPTY_INDEX_PAGE_CANARY = 0xDCDC; 30 | 31 | const int INDEX_SLOT_SIZE = 424; 32 | const int NUM_INDEX_SLOT = 18; 33 | const int MAX_NUM_CHILD_PER_NODE = 20; 34 | const int MIN_NUM_CHILD_PER_NODE = (MAX_NUM_CHILD_PER_NODE + 1) / 2; 35 | const int MAX_NUM_ENTRY_PER_NODE = MAX_NUM_CHILD_PER_NODE - 1; 36 | const int MIN_NUM_ENTRY_PER_NODE = MIN_NUM_CHILD_PER_NODE - 1; 37 | static_assert(NUM_INDEX_SLOT * INDEX_SLOT_SIZE <= PAGE_SIZE); 38 | static_assert(MAX_NUM_CHILD_PER_NODE + MAX_NUM_ENTRY_PER_NODE + 3 <= 39 | NUM_BUFFER_PAGE); 40 | 41 | const float EQUAL_PRECISION = std::numeric_limits::epsilon(); 42 | 43 | // ==== Index ==== 44 | const int INDEX_SIZE = 4; 45 | 46 | } // namespace Internal 47 | } // namespace SimpleDB 48 | 49 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/PageFile.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_PAGE_FILE_H 2 | #define _SIMPLEDB_PAGE_FILE_H 3 | 4 | #include "internal/FileCoordinator.h" 5 | 6 | namespace SimpleDB { 7 | namespace Internal { 8 | 9 | // The PageFile interfaces. 10 | namespace PF { 11 | 12 | inline void create(const std::string &fileName) { 13 | FileCoordinator::shared.createFile(fileName); 14 | } 15 | 16 | inline FileDescriptor open(const std::string &fileName) { 17 | return FileCoordinator::shared.openFile(fileName); 18 | } 19 | 20 | inline void close(FileDescriptor fd) { FileCoordinator::shared.closeFile(fd); } 21 | 22 | inline void remove(const std::string &fileName) { 23 | FileCoordinator::shared.deleteFile(fileName); 24 | } 25 | 26 | inline PageHandle getHandle(FileDescriptor fd, int page) { 27 | return FileCoordinator::shared.getHandle(fd, page); 28 | } 29 | 30 | inline char *load(PageHandle *handle) { 31 | return FileCoordinator::shared.load(handle); 32 | } 33 | 34 | inline char *loadRaw(const PageHandle &handle) { 35 | return FileCoordinator::shared.loadRaw(handle); 36 | } 37 | 38 | template 39 | inline P load(PageHandle *handle) { 40 | return reinterpret_cast

(load(handle)); 41 | } 42 | 43 | template 44 | inline P loadRaw(const PageHandle &handle) { 45 | return reinterpret_cast

(loadRaw(handle)); 46 | } 47 | 48 | inline void markDirty(const PageHandle &handle) { 49 | FileCoordinator::shared.markDirty(handle); 50 | } 51 | 52 | inline PageHandle renew(const PageHandle &handle) { 53 | return FileCoordinator::shared.renew(handle); 54 | } 55 | 56 | } // namespace PF 57 | } // namespace Internal 58 | } // namespace SimpleDB 59 | 60 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/PageHandle.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_PAGE_HANDLE_H 2 | #define _SIMPLEDB_PAGE_HANDLE_H 3 | 4 | #include "internal/CacheManager.h" 5 | 6 | namespace SimpleDB { 7 | namespace Internal { 8 | 9 | // A handle of a page cache used to access a page. 10 | struct PageHandle { 11 | friend class CacheManager; 12 | 13 | public: 14 | // This constructor is only for declaration, and must be initialized before 15 | // use. 16 | PageHandle() = default; 17 | bool validate() const { return cache->generation == generation; } 18 | 19 | #if !TESTING 20 | private: 21 | #else 22 | public: 23 | #endif 24 | PageHandle(CacheManager::PageCache *cache) 25 | : generation(cache->generation), cache(cache) {} 26 | int generation = -1; 27 | CacheManager::PageCache *cache; 28 | }; 29 | 30 | } // namespace Internal 31 | } // namespace SimpleDB 32 | 33 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/ParseHelper.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_PARSE_HELPER_H 2 | #define _SIMPLEDB_PARSE_HELPER_H 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "Error.h" 11 | #include "internal/Column.h" 12 | #include "internal/Macros.h" 13 | #include "internal/QueryFilter.h" 14 | #include "internal/Table.h" 15 | 16 | namespace SimpleDB { 17 | namespace Internal { 18 | 19 | class ParseHelper { 20 | public: 21 | static void parseName(const std::string &name, char *dest) { 22 | if (name.size() >= MAX_COLUMN_NAME_LEN) { 23 | throw Error::IncompatableValueError("Column name too long"); 24 | } 25 | std::strcpy(dest, name.c_str()); 26 | } 27 | 28 | static DataType parseDataType(const std::string &type) { 29 | if (type == "INT") { 30 | return INT; 31 | } else if (type == "FLOAT") { 32 | return FLOAT; 33 | } else if (type.compare(0, sizeof("VARCHAR") - 1, "VARCHAR") == 0) { 34 | return VARCHAR; 35 | } 36 | assert(false); 37 | return INT; 38 | } 39 | 40 | static void parseDefaultValue(const std::string &value, DataType type, 41 | ColumnValue &dest, int size) { 42 | switch (type) { 43 | case INT: 44 | dest.intValue = parseInt(value); 45 | break; 46 | case FLOAT: 47 | dest.floatValue = parseFloat(value); 48 | break; 49 | case VARCHAR: 50 | parseString(value, std::min(MAX_VARCHAR_LEN, size), 51 | dest.stringValue); 52 | break; 53 | } 54 | } 55 | 56 | static int parseInt(const std::string &value) { 57 | try { 58 | return std::stoi(value); 59 | } catch (std::exception &e) { 60 | throw Error::IncompatableValueError("Invalid integer value"); 61 | } 62 | } 63 | 64 | static float parseFloat(const std::string &value) { 65 | try { 66 | return std::stof(value); 67 | } catch (std::exception &e) { 68 | throw Error::IncompatableValueError("Invalid float value"); 69 | } 70 | } 71 | 72 | static void parseString(const std::string &value, int maxSize, char *dest) { 73 | // Here we deal with the raw string from antlr, which starts and ends 74 | // with a single quote ('). 75 | if (value.size() - 2 >= maxSize) { 76 | throw Error::IncompatableValueError("VARCHAR too long"); 77 | } 78 | std::strcpy(dest, value.substr(1, value.size() - 2).c_str()); 79 | } 80 | 81 | static CompareOp parseCompareOp(const std::string &op) { 82 | if (op == "=") { 83 | return EQ; 84 | } else if (op == "<>") { 85 | return NE; 86 | } else if (op == "<") { 87 | return LT; 88 | } else if (op == "<=") { 89 | return LE; 90 | } else if (op == ">") { 91 | return GT; 92 | } else if (op == ">=") { 93 | return GE; 94 | } 95 | assert(false); 96 | } 97 | 98 | static Column parseColumnValue(SQLParser::SqlParser::ValueContext *ctx) { 99 | Column column; 100 | if (ctx->Integer() != nullptr) { 101 | column.data.intValue = parseInt(ctx->Integer()->getText()); 102 | column.type = INT; 103 | } else if (ctx->Float() != nullptr) { 104 | column.data.floatValue = parseFloat(ctx->Float()->getText()); 105 | column.type = FLOAT; 106 | } else if (ctx->String() != nullptr) { 107 | parseString(ctx->String()->getText(), MAX_VARCHAR_LEN, 108 | column.data.stringValue); 109 | column.type = VARCHAR; 110 | } else if (ctx->Null() != nullptr) { 111 | column.isNull = true; 112 | // The data type is unknown here, must be set outside. 113 | } 114 | return column; 115 | } 116 | 117 | // static CompareValueCondition parseCompareValueCondition( 118 | // const std::string &columnName, const std::string operator_, 119 | // SQLParser::SqlParser::ValueContext *ctx) { 120 | // return CompareValueCondition(columnName, parseCompareOp(operator_), 121 | // parseColumnValue(ctx).data); 122 | // } 123 | }; 124 | 125 | } // namespace Internal 126 | } // namespace SimpleDB 127 | 128 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/ParseTreeVisitor.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_PARSE_TREE_VISITOR_H 2 | #define _SIMPLEDB_PARSE_TREE_VISITOR_H 3 | 4 | #include 5 | 6 | #include "SQLParser/SqlBaseVisitor.h" 7 | #include "SimpleDBService/query.pb.h" 8 | 9 | namespace SimpleDB { 10 | 11 | class DBMS; 12 | 13 | namespace Internal { 14 | 15 | class ParseTreeVisitor : public SQLParser::SqlBaseVisitor { 16 | public: 17 | ParseTreeVisitor() = default; 18 | ParseTreeVisitor(::SimpleDB::DBMS *dbms); 19 | 20 | virtual antlrcpp::Any visitProgram( 21 | SQLParser::SqlParser::ProgramContext *ctx) override; 22 | virtual antlrcpp::Any visitStatement( 23 | SQLParser::SqlParser::StatementContext *ctx) override; 24 | virtual antlrcpp::Any visitCreate_db( 25 | SQLParser::SqlParser::Create_dbContext *ctx) override; 26 | virtual antlrcpp::Any visitDrop_db( 27 | SQLParser::SqlParser::Drop_dbContext *ctx) override; 28 | virtual antlrcpp::Any visitShow_dbs( 29 | SQLParser::SqlParser::Show_dbsContext *ctx) override; 30 | virtual antlrcpp::Any visitUse_db( 31 | SQLParser::SqlParser::Use_dbContext *ctx) override; 32 | virtual antlrcpp::Any visitShow_tables( 33 | SQLParser::SqlParser::Show_tablesContext *ctx) override; 34 | virtual antlrcpp::Any visitCreate_table( 35 | SQLParser::SqlParser::Create_tableContext *ctx) override; 36 | virtual antlrcpp::Any visitDrop_table( 37 | SQLParser::SqlParser::Drop_tableContext *ctx) override; 38 | virtual antlrcpp::Any visitDescribe_table( 39 | SQLParser::SqlParser::Describe_tableContext *ctx) override; 40 | 41 | virtual antlrcpp::Any visitField_list( 42 | SQLParser::SqlParser::Field_listContext *ctx) override; 43 | virtual antlrcpp::Any visitNormal_field( 44 | SQLParser::SqlParser::Normal_fieldContext *ctx) override; 45 | 46 | virtual antlrcpp::Any visitAlter_table_drop_pk( 47 | SQLParser::SqlParser::Alter_table_drop_pkContext *ctx) override; 48 | virtual antlrcpp::Any visitAlter_table_add_pk( 49 | SQLParser::SqlParser::Alter_table_add_pkContext *ctx) override; 50 | virtual antlrcpp::Any visitAlter_table_drop_foreign_key( 51 | SQLParser::SqlParser::Alter_table_drop_foreign_keyContext *ctx) 52 | override; 53 | virtual antlrcpp::Any visitAlter_table_add_foreign_key( 54 | SQLParser::SqlParser::Alter_table_add_foreign_keyContext *ctx) override; 55 | virtual antlrcpp::Any visitAlter_add_index( 56 | SQLParser::SqlParser::Alter_add_indexContext *ctx) override; 57 | virtual antlrcpp::Any visitAlter_drop_index( 58 | SQLParser::SqlParser::Alter_drop_indexContext *ctx) override; 59 | virtual antlrcpp::Any visitShow_indexes( 60 | SQLParser::SqlParser::Show_indexesContext *ctx) override; 61 | 62 | virtual antlrcpp::Any visitWhere_and_clause( 63 | SQLParser::SqlParser::Where_and_clauseContext *ctx) override; 64 | virtual antlrcpp::Any visitWhere_operator_expression( 65 | SQLParser::SqlParser::Where_operator_expressionContext *ctx) override; 66 | virtual antlrcpp::Any visitWhere_null( 67 | SQLParser::SqlParser::Where_nullContext *ctx) override; 68 | virtual antlrcpp::Any visitColumn( 69 | SQLParser::SqlParser::ColumnContext *ctx) override; 70 | 71 | virtual antlrcpp::Any visitInsert_into_table( 72 | SQLParser::SqlParser::Insert_into_tableContext *ctx) override; 73 | virtual antlrcpp::Any visitSelect_table_( 74 | SQLParser::SqlParser::Select_table_Context *ctx) override; 75 | virtual antlrcpp::Any visitUpdate_table( 76 | SQLParser::SqlParser::Update_tableContext *ctx) override; 77 | virtual antlrcpp::Any visitDelete_from_table( 78 | SQLParser::SqlParser::Delete_from_tableContext *ctx) override; 79 | 80 | private: 81 | DBMS *dbms; 82 | 83 | #define DECLARE_WRAPPER(upperName, lowerName) \ 84 | Service::ExecutionResult wrap(const Service::upperName##Result &result) { \ 85 | Service::ExecutionResult execResult; \ 86 | execResult.mutable_##lowerName()->CopyFrom(result); \ 87 | return execResult; \ 88 | } 89 | 90 | DECLARE_WRAPPER(Plain, plain); 91 | DECLARE_WRAPPER(ShowDatabases, show_databases); 92 | DECLARE_WRAPPER(ShowTable, show_table); 93 | DECLARE_WRAPPER(DescribeTable, describe_table); 94 | DECLARE_WRAPPER(ShowIndexes, show_indexes); 95 | DECLARE_WRAPPER(Query, query); 96 | 97 | #undef DECLARE_WRAPPER 98 | }; 99 | 100 | } // namespace Internal 101 | } // namespace SimpleDB 102 | 103 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/QueryBuilder.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_QUERY_BUILDER_H 2 | #define _SIMPLEDB_QUERY_BUILDER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "internal/QueryDataSource.h" 9 | #include "internal/QueryFilter.h" 10 | 11 | namespace SimpleDB { 12 | namespace Internal { 13 | 14 | // Build a query schedule using a series of selectors. 15 | class QueryBuilder : public QueryDataSource { 16 | public: 17 | using Result = std::vector>; 18 | QueryBuilder(QueryDataSource *dataSource); 19 | QueryBuilder(std::shared_ptr dataSource); 20 | QueryBuilder() = default; 21 | 22 | QueryBuilder &condition(const CompareValueCondition &condition); 23 | QueryBuilder &condition(const std::string &columnName, CompareOp op, 24 | const char *string); 25 | QueryBuilder &condition(const std::string &columnName, CompareOp op, 26 | int value); 27 | QueryBuilder &condition(const ColumnId &columnId, CompareOp op, 28 | const char *string); 29 | QueryBuilder &condition(const ColumnId &columnId, CompareOp op, 30 | const ColumnValue &value); 31 | 32 | QueryBuilder &condition(const CompareColumnCondition &condition); 33 | QueryBuilder &nullCondition(const CompareNullCondition &condition); 34 | QueryBuilder &nullCondition(const std::string &columnName, bool isNull); 35 | QueryBuilder &nullCondition(const ColumnId &columnId, bool isNull); 36 | QueryBuilder &select(const QuerySelector &selector); 37 | QueryBuilder &select(const std::string &column); 38 | QueryBuilder &select(const ColumnId &id); 39 | QueryBuilder &limit(int count); 40 | QueryBuilder &offset(int offset); 41 | 42 | [[nodiscard]] Result execute(); 43 | 44 | // QueryDataSource requirements, allowing chained pipelines. 45 | virtual void iterate(IterateCallback callback) override; 46 | virtual std::vector getColumnInfo() override; 47 | 48 | bool validForUpdateOrDelete() const; 49 | QueryDataSource *getDataSource(); 50 | 51 | private: 52 | std::vector valueConditionFilters; 53 | std::vector columnConditionFilters; 54 | std::vector nullConditionFilters; 55 | SelectFilter selectFilter; 56 | LimitFilter limitFilter; 57 | OffsetFilter offsetFilter; 58 | 59 | QueryDataSource *dataSourceRawPtr = nullptr; 60 | std::shared_ptr dataSourceSharedPtr; 61 | bool isRaw; 62 | VirtualTable virtualTable; 63 | 64 | void checkDataSource(); 65 | AggregatedFilter aggregateAllFilters(); 66 | }; 67 | 68 | } // namespace Internal 69 | } // namespace SimpleDB 70 | 71 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/QueryDataSource.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_QUERY_DATASOURCE_H 2 | #define _SIMPLEDB_QUERY_DATASOURCE_H 3 | 4 | #include 5 | #include 6 | 7 | #include "internal/Column.h" 8 | 9 | namespace SimpleDB { 10 | namespace Internal { 11 | 12 | class QueryDataSource { 13 | public: 14 | using IterateCallback = std::function; 15 | virtual ~QueryDataSource() = default; 16 | virtual void iterate(IterateCallback callback) = 0; 17 | virtual std::vector getColumnInfo() = 0; 18 | virtual bool acceptCondition( 19 | const struct CompareValueCondition &condition) { 20 | return false; 21 | } 22 | }; 23 | 24 | } // namespace Internal 25 | } // namespace SimpleDB 26 | 27 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/QueryFilter.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_QUERY_FILTER 2 | #define _SIMPLEDB_QUERY_FILTER 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "internal/Table.h" 12 | 13 | namespace SimpleDB { 14 | namespace Internal { 15 | 16 | enum CompareOp { 17 | EQ, // = 18 | NE, // <> 19 | LT, // < 20 | LE, // <= 21 | GT, // > 22 | GE, // >= 23 | }; 24 | 25 | struct ColumnId { 26 | std::string tableName; 27 | std::string columnName; 28 | 29 | std::string getDesc() const { 30 | return tableName.empty() ? columnName : tableName + "." + columnName; 31 | } 32 | }; 33 | 34 | struct CompareValueCondition { 35 | ColumnId columnId; 36 | CompareOp op; 37 | ColumnValue value; 38 | 39 | CompareValueCondition(const ColumnId &id, CompareOp op, 40 | const ColumnValue &value) 41 | : columnId(id), op(op), value(value) {} 42 | 43 | CompareValueCondition(const ColumnId &id, CompareOp op, const char *string) 44 | : columnId(id), op(op) { 45 | std::strcpy(value.stringValue, string); 46 | } 47 | 48 | CompareValueCondition() = default; 49 | 50 | // static CompareValueCondition eq(const char *columnName, 51 | // const ColumnValue &value) { 52 | // return CompareValueCondition(columnName, EQ, value); 53 | // } 54 | 55 | // static CompareValueCondition eq(const char *columnName, 56 | // const char *string) { 57 | // return CompareValueCondition(columnName, EQ, string); 58 | // } 59 | }; 60 | 61 | struct CompareNullCondition { 62 | ColumnId columnId; 63 | bool isNull; 64 | CompareNullCondition(const ColumnId &id, bool isNull) 65 | : columnId(id), isNull(isNull) {} 66 | CompareNullCondition() = default; 67 | }; 68 | 69 | struct CompareColumnCondition { 70 | ColumnId lhs; 71 | CompareOp op; 72 | ColumnId rhs; 73 | CompareColumnCondition(const ColumnId &lhs, CompareOp op, 74 | const ColumnId &rhs) 75 | : lhs(lhs), op(op), rhs(rhs) {} 76 | CompareColumnCondition() = default; 77 | }; 78 | 79 | struct QuerySelector { 80 | enum Type { COLUMN, COUNT_STAR, COUNT_COL, AVG, MAX, MIN, SUM }; 81 | Type type; 82 | ColumnId column; 83 | 84 | std::string getColumnName() const; 85 | }; 86 | 87 | struct VirtualTable { 88 | VirtualTable() = default; 89 | VirtualTable(const std::vector &columns) { 90 | this->columns = columns; 91 | } 92 | 93 | std::vector columns; 94 | 95 | int getColumnIndex(const ColumnId id) { 96 | int index = -1; 97 | for (int i = 0; i < columns.size(); i++) { 98 | if (columns[i].columnName == id.columnName) { 99 | if (id.tableName.empty() || 100 | columns[i].tableName == id.tableName) { 101 | if (index != -1) { 102 | throw AmbiguousColumnError(id.getDesc()); 103 | } 104 | index = i; 105 | } 106 | } 107 | } 108 | return index; 109 | } 110 | }; 111 | 112 | struct BaseFilter { 113 | virtual ~BaseFilter() {} 114 | virtual std::pair apply(Columns &columns) = 0; 115 | virtual void build(){}; 116 | virtual bool finalize(Columns &columns) { return false; } 117 | }; 118 | 119 | struct ValueConditionFilter : public BaseFilter { 120 | ValueConditionFilter() = default; 121 | ~ValueConditionFilter() = default; 122 | virtual void build() override; 123 | virtual std::pair apply(Columns &columns) override; 124 | CompareValueCondition condition; 125 | VirtualTable *table; 126 | int columnIndex; 127 | }; 128 | 129 | struct NullConditionFilter : public BaseFilter { 130 | NullConditionFilter() = default; 131 | ~NullConditionFilter() = default; 132 | virtual void build() override; 133 | virtual std::pair apply(Columns &columns) override; 134 | CompareNullCondition condition; 135 | VirtualTable *table; 136 | int columnIndex; 137 | }; 138 | 139 | struct ColumnConditionFilter : public BaseFilter { 140 | ColumnConditionFilter() = default; 141 | ~ColumnConditionFilter() = default; 142 | virtual void build() override; 143 | virtual std::pair apply(Columns &columns) override; 144 | CompareColumnCondition condition; 145 | VirtualTable *table; 146 | int columnIndex1; 147 | int columnIndex2; 148 | }; 149 | 150 | struct SelectFilter : public BaseFilter { 151 | struct Context { 152 | union { 153 | int intValue; 154 | float floatValue; 155 | } value; 156 | bool isNull = true; 157 | int count = 0; 158 | void initializeInt(int value) { 159 | if (isNull) { 160 | isNull = false; 161 | this->value.intValue = value; 162 | } 163 | } 164 | void initializeFloat(float value) { 165 | if (isNull) { 166 | isNull = false; 167 | this->value.floatValue = value; 168 | } 169 | } 170 | }; 171 | SelectFilter() = default; 172 | ~SelectFilter() = default; 173 | virtual std::pair apply(Columns &columns) override; 174 | void build() override; 175 | virtual bool finalize(Columns &columns) override; 176 | std::vector selectors; 177 | std::vector selectIndexes; 178 | std::vector selectContexts; 179 | bool isAggregated; 180 | VirtualTable *table; 181 | }; 182 | 183 | struct LimitFilter : public BaseFilter { 184 | LimitFilter() : limit(-1) {} 185 | ~LimitFilter() = default; 186 | virtual std::pair apply(Columns &columns) override; 187 | int limit; 188 | int count = 0; 189 | }; 190 | 191 | struct OffsetFilter : public BaseFilter { 192 | OffsetFilter() : offset(0) {} 193 | ~OffsetFilter() = default; 194 | virtual std::pair apply(Columns &columns) override; 195 | int offset; 196 | int count = 0; 197 | }; 198 | 199 | struct AggregatedFilter : public BaseFilter { 200 | AggregatedFilter() = default; 201 | virtual std::pair apply(Columns &columns) override; 202 | virtual bool finalize(Columns &columns) override; 203 | virtual void build() override; 204 | std::vector filters; 205 | }; 206 | 207 | } // namespace Internal 208 | } // namespace SimpleDB 209 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Service.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_SERVICE_H 2 | #define _SIMPLEDB_SERVICE_H 3 | 4 | #include "SimpleDBService/query.pb.h" 5 | // Undef the evil PAGE_SIZE macro somewhere deep inside this header. 6 | #undef PAGE_SIZE 7 | 8 | #endif -------------------------------------------------------------------------------- /SimpleDB/include/SimpleDB/internal/Table.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDB_TABLE_H 2 | #define _SIMPLEDB_TABLE_H 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "internal/Column.h" 12 | #include "internal/FileCoordinator.h" 13 | #include "internal/Macros.h" 14 | #include "internal/QueryDataSource.h" 15 | 16 | namespace SimpleDB { 17 | class DBMS; 18 | 19 | namespace Internal { 20 | 21 | struct ForeignKey { 22 | std::string name; 23 | std::string table; 24 | std::string ref; 25 | DataType type; // The referencing column's type, for validation. 26 | }; 27 | 28 | struct ColumnMeta { 29 | DataType type; 30 | ColumnSizeType size; 31 | bool nullable; 32 | char name[MAX_COLUMN_NAME_LEN]; 33 | 34 | bool hasDefault; 35 | ColumnValue defaultValue; 36 | 37 | std::string typeDesc() const; 38 | std::string defaultValDesc() const; 39 | }; 40 | 41 | // A Table holds the metadata of a certain table, which should be unique 42 | // thourghout the program, and be stored in memory once created for the sake of 43 | // metadata reading/writing performance. 44 | class Table : public QueryDataSource { 45 | friend class QueryBuilder; 46 | friend class ::SimpleDB::DBMS; 47 | friend class IndexedTable; 48 | 49 | public: 50 | // The metadata is not initialized in this constructor. 51 | Table() = default; 52 | ~Table(); 53 | 54 | // Open the table from a file, which must be created by `create()` before. 55 | void open(const std::string &file) noexcept(false); 56 | 57 | // Create a new table in a file. 58 | void create( 59 | const std::string &file, const std::string &name, 60 | const std::vector &columns, 61 | const std::string &primaryKey = {}, 62 | const std::vector &foreignKeys = {}) noexcept(false); 63 | 64 | // Get record. 65 | [[nodiscard]] Columns get(RecordID id, 66 | ColumnBitmap columnBitmap = COLUMN_BITMAP_ALL); 67 | void get(RecordID id, Columns &columns, 68 | ColumnBitmap columnBitmap = COLUMN_BITMAP_ALL); 69 | 70 | // Insert record, returns (page, slot) of the inserted record. 71 | RecordID insert(const Columns &values, 72 | ColumnBitmap bitmap = COLUMN_BITMAP_ALL); 73 | 74 | // Update record. 75 | void update(RecordID id, const Columns &columns, 76 | ColumnBitmap columnBitmap = COLUMN_BITMAP_ALL); 77 | 78 | // Remove record. 79 | void remove(RecordID id); 80 | 81 | // Set primary key. 82 | void setPrimaryKey(const std::string &field); 83 | void dropPrimaryKey(const std::string &field); 84 | 85 | void close(); 86 | 87 | int getColumnIndex(const char *name) const; 88 | std::string getColumnName(int index) const; 89 | 90 | // QueryDataSource requirements. 91 | virtual void iterate(IterateCallback callback) override; 92 | virtual std::vector getColumnInfo() override; 93 | 94 | #if !TESTING 95 | private: 96 | #endif 97 | struct TableMeta { 98 | // Keep first. 99 | uint16_t headCanary = TABLE_META_CANARY; 100 | 101 | char name[MAX_TABLE_NAME_LEN + 1]; 102 | 103 | uint32_t numColumn; 104 | ColumnMeta columns[MAX_COLUMNS]; 105 | int primaryKeyIndex; 106 | ForeignKey foreignKeys[MAX_FOREIGN_KEYS]; 107 | uint16_t numUsedPages; 108 | uint16_t firstFree; 109 | int recordSize; 110 | 111 | // Keep last. 112 | uint16_t tailCanary = TABLE_META_CANARY; 113 | }; 114 | 115 | struct PageMeta { 116 | // Keep first. 117 | uint16_t headCanary = PAGE_META_CANARY; 118 | 119 | using BitmapType = int64_t; 120 | BitmapType occupied = 0; 121 | 122 | static_assert(sizeof(BitmapType) * 8 >= MAX_SLOT_PER_PAGE); 123 | 124 | uint16_t nextFree; 125 | 126 | // Keep last. 127 | uint16_t tailCanary = PAGE_META_CANARY; 128 | }; 129 | 130 | // The metadata should be fit into the first slot. 131 | static_assert(sizeof(TableMeta) < PAGE_SIZE); 132 | static_assert(sizeof(PageMeta) < PAGE_SIZE); 133 | 134 | struct RecordMeta { 135 | ColumnBitmap nullBitmap; 136 | }; 137 | 138 | bool initialized = false; 139 | FileDescriptor fd; 140 | // TODO: Pin meta page? 141 | TableMeta meta; 142 | std::map pageHandleMap; 143 | std::map columnNameMap; 144 | 145 | void checkInit() noexcept(false); 146 | void flushMeta() noexcept(false); 147 | void flushPageMeta(int page, const PageMeta &meta); 148 | 149 | PageHandle *getHandle(int page); 150 | 151 | void deserialize(const char *srcData, Columns &destObjects, 152 | ColumnBitmap columnBitmap); 153 | void serialize(const Columns &srcObjects, char *destData, ColumnBitmap map, 154 | bool all); 155 | 156 | // Must ensure that the handle is valid. 157 | bool occupied(const PageHandle &handle, int slot); 158 | 159 | // Side effect: might create a new page, thus modifying meta, 160 | // firstFreePagMeta, etc. 161 | RecordID getEmptySlot(); 162 | 163 | int slotSize(); 164 | int numSlotPerPage(); 165 | 166 | bool isPageFull(PageMeta *pageMeta); 167 | 168 | void validateSlot(int page, int slot); 169 | void validateColumnBitmap(const Columns &columns, ColumnBitmap bitmap, 170 | bool isUpdate); 171 | }; 172 | 173 | } // namespace Internal 174 | } // namespace SimpleDB 175 | #endif -------------------------------------------------------------------------------- /SimpleDB/src/backend/IndexedTable.cc: -------------------------------------------------------------------------------- 1 | #include "internal/IndexedTable.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace SimpleDB { 10 | namespace Internal { 11 | 12 | IndexedTable::IndexedTable(Table *table, GetIndexFunc getIndex) 13 | : table(table), getIndex(getIndex) {} 14 | 15 | void IndexedTable::iterate(IterateCallback callback) { 16 | collapseRanges(); 17 | 18 | if (emptySet) { 19 | return; 20 | } 21 | 22 | if (index == nullptr) { 23 | return table->iterate(callback); 24 | } 25 | 26 | Columns columns; 27 | 28 | for (auto &range : ranges) { 29 | index->iterateRange(range, [&](RecordID id) { 30 | table->get(id, columns); 31 | return callback(id, columns); 32 | }); 33 | } 34 | } 35 | 36 | bool IndexedTable::acceptCondition(const CompareValueCondition &condition) { 37 | if (!condition.columnId.tableName.empty() && 38 | condition.columnId.tableName != table->meta.name) { 39 | return false; 40 | } 41 | int columnIndex = 42 | table->getColumnIndex(condition.columnId.columnName.c_str()); 43 | if (columnIndex < 0) { 44 | return false; 45 | } 46 | const ColumnMeta &column = table->meta.columns[columnIndex]; 47 | 48 | if (column.type != INT) { 49 | return false; 50 | } 51 | 52 | if (index == nullptr) { 53 | index = getIndex(table->meta.name, condition.columnId.columnName); 54 | columnName = condition.columnId.columnName; 55 | 56 | if (index == nullptr) { 57 | return false; 58 | } 59 | } else if (columnName != condition.columnId.columnName) { 60 | // Only taking the first (indexed) column from the conditions. 61 | return false; 62 | } 63 | 64 | ranges.push_back(makeRange(condition)); 65 | 66 | return true; 67 | } 68 | 69 | std::vector IndexedTable::getColumnInfo() { 70 | return table->getColumnInfo(); 71 | } 72 | 73 | Table *IndexedTable::getTable() { return table; } 74 | 75 | Index::Range IndexedTable::makeRange(const CompareValueCondition &condition) { 76 | int value = condition.value.intValue; 77 | 78 | switch (condition.op) { 79 | case EQ: 80 | return {value, value}; 81 | case NE: 82 | return {value + 1, value - 1}; 83 | case LT: 84 | return {INT_MIN, value - 1}; 85 | case LE: 86 | return {INT_MIN, value}; 87 | case GT: 88 | return {value + 1, INT_MAX}; 89 | case GE: 90 | return {value, INT_MAX}; 91 | } 92 | } 93 | 94 | void IndexedTable::collapseRanges() { 95 | Index::Range collapsedRange = {INT_MIN, INT_MAX}; 96 | std::vector neValues; 97 | 98 | for (const auto &range : ranges) { 99 | if (range.first > range.second) { 100 | assert(range.first == range.second + 2); 101 | neValues.push_back(range.first - 1); 102 | continue; 103 | } 104 | 105 | collapsedRange.first = std::max(collapsedRange.first, range.first); 106 | collapsedRange.second = std::min(collapsedRange.second, range.second); 107 | 108 | if (collapsedRange.first > collapsedRange.second) { 109 | emptySet = true; 110 | return; 111 | } 112 | } 113 | 114 | ranges.clear(); 115 | 116 | std::sort(neValues.begin(), neValues.end()); 117 | 118 | for (auto neVal : neValues) { 119 | if (neVal < collapsedRange.first || neVal > collapsedRange.second) { 120 | continue; 121 | } 122 | 123 | if (neVal == collapsedRange.first) { 124 | collapsedRange.first++; 125 | } else if (neVal == collapsedRange.second) { 126 | collapsedRange.second--; 127 | } else { 128 | ranges.push_back({collapsedRange.first, neVal - 1}); 129 | collapsedRange.first = neVal + 1; 130 | } 131 | 132 | if (collapsedRange.first > collapsedRange.second) { 133 | emptySet = true; 134 | return; 135 | } 136 | } 137 | 138 | ranges.push_back(collapsedRange); 139 | } 140 | 141 | } // namespace Internal 142 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/backend/JoinedTable.cc: -------------------------------------------------------------------------------- 1 | #include "internal/JoinedTable.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace SimpleDB { 8 | namespace Internal { 9 | 10 | void JoinedTable::append(std::shared_ptr table) { 11 | assert(tables.size() < 2); 12 | tables.push_back(table); 13 | } 14 | 15 | bool JoinedTable::acceptCondition(const CompareValueCondition &condition) { 16 | for (auto table : tables) { 17 | if (table->acceptCondition(condition)) { 18 | return true; 19 | } 20 | } 21 | return false; 22 | } 23 | 24 | std::vector JoinedTable::getColumnInfo() { 25 | std::vector result; 26 | for (auto table : tables) { 27 | auto columns = table->getColumnInfo(); 28 | result.insert(result.end(), columns.begin(), columns.end()); 29 | } 30 | return result; 31 | } 32 | 33 | void JoinedTable::iterate(IterateCallback callback) { 34 | // Only support <= 2 tables. 35 | if (tables.size() == 1) { 36 | tables[0]->iterate(callback); 37 | } else if (tables.size() == 2) { 38 | // A simple nested, pipelined loop join. 39 | // TODO: Decide join order based on table sizes, indexes... 40 | tables[0]->iterate([&](RecordID id, const Columns &columns1) { 41 | bool continue_ = true; 42 | tables[1]->iterate([&](RecordID id, const Columns &columns2) { 43 | Columns columns; 44 | columns.insert(columns.end(), columns1.begin(), columns1.end()); 45 | columns.insert(columns.end(), columns2.begin(), columns2.end()); 46 | continue_ = callback(id, columns); 47 | return continue_; 48 | }); 49 | return continue_; 50 | }); 51 | } 52 | } 53 | 54 | } // namespace Internal 55 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/backend/QueryBuilder.cc: -------------------------------------------------------------------------------- 1 | #include "internal/QueryBuilder.h" 2 | 3 | #include 4 | 5 | #include "Error.h" 6 | #include "internal/Column.h" 7 | #include "internal/Comparer.h" 8 | #include "internal/Logger.h" 9 | #include "internal/QueryFilter.h" 10 | 11 | namespace SimpleDB { 12 | namespace Internal { 13 | 14 | QueryBuilder::QueryBuilder(QueryDataSource *source) 15 | : dataSourceRawPtr(source), isRaw(true) {} 16 | 17 | QueryBuilder::QueryBuilder(std::shared_ptr source) 18 | : dataSourceSharedPtr(source), isRaw(false) {} 19 | 20 | QueryDataSource *QueryBuilder::getDataSource() { 21 | return isRaw ? dataSourceRawPtr : dataSourceSharedPtr.get(); 22 | } 23 | 24 | QueryBuilder &QueryBuilder::condition(const CompareValueCondition &condition) { 25 | if (getDataSource()->acceptCondition(condition)) { 26 | return *this; 27 | } 28 | ValueConditionFilter filter; 29 | filter.condition = condition; 30 | valueConditionFilters.push_back(filter); 31 | return *this; 32 | } 33 | 34 | QueryBuilder &QueryBuilder::condition(const ColumnId &columnId, CompareOp op, 35 | const ColumnValue &value) { 36 | return condition(CompareValueCondition{columnId, op, value}); 37 | } 38 | 39 | QueryBuilder &QueryBuilder::condition(const std::string &columnName, 40 | CompareOp op, const char *string) { 41 | return condition(ColumnId{.columnName = columnName.c_str()}, op, string); 42 | } 43 | 44 | QueryBuilder &QueryBuilder::condition(const std::string &columnName, 45 | CompareOp op, int value) { 46 | CompareValueCondition cond; 47 | cond.value = ColumnValue{.intValue = value}; 48 | cond.op = op; 49 | cond.columnId = {.columnName = columnName}; 50 | return condition(cond); 51 | } 52 | 53 | QueryBuilder &QueryBuilder::condition(const ColumnId &id, CompareOp op, 54 | const char *string) { 55 | return condition(CompareValueCondition(id, op, string)); 56 | } 57 | 58 | QueryBuilder &QueryBuilder::condition(const CompareColumnCondition &condition) { 59 | ColumnConditionFilter filter; 60 | filter.condition = condition; 61 | columnConditionFilters.push_back(filter); 62 | return *this; 63 | } 64 | 65 | QueryBuilder &QueryBuilder::nullCondition( 66 | const CompareNullCondition &condition) { 67 | NullConditionFilter filter; 68 | filter.condition = condition; 69 | nullConditionFilters.push_back(filter); 70 | return *this; 71 | } 72 | 73 | QueryBuilder &QueryBuilder::nullCondition(const std::string &columnName, 74 | bool isNull) { 75 | return nullCondition({.columnName = columnName.c_str()}, isNull); 76 | } 77 | 78 | QueryBuilder &QueryBuilder::nullCondition(const ColumnId &id, bool isNull) { 79 | return nullCondition(CompareNullCondition(id, isNull)); 80 | } 81 | 82 | QueryBuilder &QueryBuilder::select(const QuerySelector &selector) { 83 | selectFilter.selectors.push_back(selector); 84 | return *this; 85 | } 86 | 87 | QueryBuilder &QueryBuilder::select(const std::string &column) { 88 | return select(ColumnId{.columnName = column.c_str()}); 89 | } 90 | 91 | QueryBuilder &QueryBuilder::select(const ColumnId &id) { 92 | selectFilter.selectors.push_back({QuerySelector::COLUMN, id}); 93 | return *this; 94 | } 95 | 96 | QueryBuilder &QueryBuilder::limit(int count) { 97 | if (limitFilter.limit != -1) { 98 | Logger::log(WARNING, 99 | "QueryBuilder: limit is already set (%d) and is " 100 | "overwritten by %d", 101 | limitFilter.limit, count); 102 | } 103 | if (count < 0) { 104 | throw InvalidLimitError(); 105 | } 106 | limitFilter.limit = count; 107 | return *this; 108 | } 109 | 110 | QueryBuilder &QueryBuilder::offset(int offset) { 111 | offsetFilter.offset = offset; 112 | return *this; 113 | } 114 | 115 | QueryBuilder::Result QueryBuilder::execute() { 116 | Result result; 117 | 118 | this->iterate([&](RecordID rid, Columns &columns) { 119 | result.emplace_back(rid, columns); 120 | return true; 121 | }); 122 | 123 | return result; 124 | } 125 | 126 | // TODO: Add tests for this. 127 | void QueryBuilder::iterate(IterateCallback callback) { 128 | checkDataSource(); 129 | 130 | // TODO: Check if is all about COUNT(*) 131 | 132 | AggregatedFilter filter = aggregateAllFilters(); 133 | 134 | getDataSource()->iterate([&](RecordID rid, Columns &columns) { 135 | auto [accept, continue_] = filter.apply(columns); 136 | if (accept) { 137 | return callback(rid, columns) && continue_; 138 | } 139 | return continue_; 140 | }); 141 | 142 | Columns columns; 143 | bool ret = filter.finalize(columns); 144 | if (ret) { 145 | callback(RecordID::NULL_RECORD, columns); 146 | } 147 | } 148 | 149 | std::vector QueryBuilder::getColumnInfo() { 150 | checkDataSource(); 151 | auto columnMetas = getDataSource()->getColumnInfo(); 152 | 153 | std::vector result; 154 | 155 | if (selectFilter.selectors.size() == 0) { 156 | return columnMetas; 157 | } 158 | 159 | for (const auto &selector : selectFilter.selectors) { 160 | if (selector.type == QuerySelector::COUNT_STAR) { 161 | result.push_back({.tableName = std::string(), /* TODO */ 162 | .columnName = selector.getColumnName(), 163 | .type = INT}); 164 | } else { 165 | int index = -1; 166 | for (int i = 0; i < columnMetas.size(); i++) { 167 | if (columnMetas[i].columnName == selector.column.columnName) { 168 | if (selector.column.tableName.empty() && index != -1) { 169 | throw AmbiguousColumnError(selector.column.getDesc()); 170 | } 171 | if (selector.column.tableName.empty() || 172 | selector.column.tableName == columnMetas[i].tableName) { 173 | index = i; 174 | } 175 | } 176 | } 177 | 178 | if (index == -1) { 179 | throw ColumnNotFoundError(selector.column.getDesc()); 180 | } 181 | if (selector.type == QuerySelector::COLUMN) { 182 | result.push_back(columnMetas[index]); 183 | } else { 184 | DataType type; 185 | switch (selector.type) { 186 | case QuerySelector::AVG: 187 | type = FLOAT; 188 | break; 189 | case QuerySelector::COUNT_COL: 190 | case QuerySelector::COUNT_STAR: 191 | type = INT; 192 | default: 193 | type = columnMetas[index].type; 194 | } 195 | result.push_back({.tableName = std::string(), /* TODO */ 196 | .columnName = selector.getColumnName(), 197 | .type = type}); 198 | } 199 | } 200 | } 201 | 202 | return result; 203 | } 204 | 205 | bool QueryBuilder::validForUpdateOrDelete() const { 206 | return selectFilter.selectors.size() == 0 && limitFilter.limit < 0 && 207 | offsetFilter.offset == 0; 208 | } 209 | 210 | void QueryBuilder::checkDataSource() { 211 | if (getDataSource() == nullptr) { 212 | throw NoScanDataSourceError(); 213 | } 214 | } 215 | 216 | AggregatedFilter QueryBuilder::aggregateAllFilters() { 217 | // Create a virtual table. 218 | virtualTable = VirtualTable(getDataSource()->getColumnInfo()); 219 | 220 | AggregatedFilter filter; 221 | // Condition filters comes before selectors. 222 | for (int i = 0; i < nullConditionFilters.size(); i++) { 223 | nullConditionFilters[i].table = &virtualTable; 224 | filter.filters.push_back(&nullConditionFilters[i]); 225 | } 226 | for (int i = 0; i < valueConditionFilters.size(); i++) { 227 | valueConditionFilters[i].table = &virtualTable; 228 | filter.filters.push_back(&valueConditionFilters[i]); 229 | } 230 | for (int i = 0; i < columnConditionFilters.size(); i++) { 231 | columnConditionFilters[i].table = &virtualTable; 232 | filter.filters.push_back(&columnConditionFilters[i]); 233 | } 234 | // The select filters comes after. 235 | if (selectFilter.selectors.size() > 0) { 236 | selectFilter.table = &virtualTable; 237 | filter.filters.push_back(&selectFilter); 238 | } 239 | 240 | // Only apply limit and offset filters when is not aggregated. 241 | if (!selectFilter.isAggregated) { 242 | // Only apply limit and offset filters when is not aggregated. 243 | filter.filters.push_back(&offsetFilter); 244 | filter.filters.push_back(&limitFilter); 245 | } 246 | 247 | filter.build(); 248 | 249 | return filter; 250 | } 251 | 252 | } // namespace Internal 253 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/backend/QueryFilter.cc: -------------------------------------------------------------------------------- 1 | #include "internal/QueryFilter.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "Error.h" 9 | #include "internal/Comparer.h" 10 | #include "internal/Logger.h" 11 | #include "internal/Table.h" 12 | 13 | namespace SimpleDB { 14 | namespace Internal { 15 | 16 | // ===== Begin AggregatedFilter ===== 17 | void AggregatedFilter::build() { 18 | for (auto &filter : filters) { 19 | filter->build(); 20 | } 21 | } 22 | 23 | std::pair AggregatedFilter::apply(Columns &columns) { 24 | bool stop = false; 25 | for (auto filter : filters) { 26 | auto [accept, continue_] = filter->apply(columns); 27 | if (!continue_) { 28 | stop = true; 29 | } 30 | if (!accept) { 31 | return {false, !stop}; 32 | } 33 | } 34 | return {true, !stop}; 35 | } 36 | bool AggregatedFilter::finalize(Columns &columns) { 37 | bool ret = false; 38 | for (auto filter : filters) { 39 | if (filter->finalize(columns)) { 40 | ret = true; 41 | } 42 | } 43 | return ret; 44 | } 45 | // ====== End AggregatedFilter ====== 46 | 47 | // ===== Begin LimitFilter ===== 48 | std::pair LimitFilter::apply(Columns &columns) { 49 | count++; 50 | if (limit < 0) { 51 | return {true, true}; 52 | } 53 | 54 | if (count > limit) { 55 | return {false, false}; 56 | } 57 | 58 | if (count == limit) { 59 | return {true, false}; 60 | } 61 | 62 | return {true, true}; 63 | } 64 | // ====== End LimitFilter ====== 65 | 66 | // ===== Begin OffsetFilter ===== 67 | std::pair OffsetFilter::apply(Columns &columns) { 68 | count++; 69 | if (count <= offset) { 70 | return {false, true}; 71 | } 72 | return {true, true}; 73 | } 74 | // ====== End OffsetFilter ====== 75 | 76 | // ===== Begin SelectFilter ===== 77 | void SelectFilter::build() { 78 | for (int i = 0; i < selectors.size(); i++) { 79 | const auto &selector = selectors[i]; 80 | if (selector.type == QuerySelector::COLUMN) { 81 | isAggregated = false; 82 | auto index = table->getColumnIndex(selector.column); 83 | if (index < 0) { 84 | throw ColumnNotFoundError(selector.column.getDesc()); 85 | } 86 | selectIndexes.push_back(index); 87 | } else { 88 | // Aggregated column. 89 | isAggregated = true; 90 | if (selector.type != QuerySelector::COUNT_STAR) { 91 | auto index = table->getColumnIndex(selector.column); 92 | if (index < 0) { 93 | throw ColumnNotFoundError(selector.column.getDesc()); 94 | } 95 | if (table->columns[index].type == VARCHAR) { 96 | throw AggregatorError( 97 | "Cannot aggregate on VARCHAR columns"); 98 | } 99 | selectIndexes.push_back(index); 100 | } else { 101 | selectIndexes.push_back(-1); 102 | } 103 | } 104 | } 105 | 106 | if (isAggregated) { 107 | selectContexts.resize(selectors.size()); 108 | } 109 | } 110 | 111 | std::pair SelectFilter::apply(Columns &columns) { 112 | Columns newColumns; 113 | // TODO: Optimization. 114 | 115 | if (!isAggregated) { 116 | for (int index : selectIndexes) { 117 | newColumns.push_back(columns[index]); 118 | } 119 | } else { 120 | for (size_t i = 0; i < selectors.size(); i++) { 121 | auto &selector = selectors[i]; 122 | auto &context = selectContexts[i]; 123 | if (selector.type == QuerySelector::COUNT_STAR) { 124 | // COUNT(*) 125 | context.initializeInt(0); 126 | context.value.intValue++; 127 | } else { 128 | assert(columns.size() == table->columns.size()); 129 | const auto &columnInfo = table->columns[selectIndexes[i]]; 130 | bool isNull = columns[selectIndexes[i]].isNull; 131 | if (!isNull) { 132 | context.count++; 133 | } 134 | 135 | #define AGGREGATE(_type, _Type) \ 136 | auto value = columns[selectIndexes[i]].data._type; \ 137 | switch (selector.type) { \ 138 | case QuerySelector::COUNT_COL: \ 139 | context.initializeInt(0); \ 140 | context.value.intValue += (isNull ? 0 : 1); \ 141 | break; \ 142 | case QuerySelector::MIN: \ 143 | if (!isNull) { \ 144 | context.initialize##_Type(value); \ 145 | context.value._type = std::min(context.value._type, value); \ 146 | } \ 147 | break; \ 148 | case QuerySelector::MAX: \ 149 | if (!isNull) { \ 150 | context.initialize##_Type(value); \ 151 | context.value._type = std::max(context.value._type, value); \ 152 | } \ 153 | break; \ 154 | case QuerySelector::SUM: \ 155 | case QuerySelector::AVG: \ 156 | if (!isNull) { \ 157 | context.initialize##_Type(0); \ 158 | context.value._type += value; \ 159 | } \ 160 | break; \ 161 | default: \ 162 | assert(false); \ 163 | } 164 | if (columnInfo.type == INT) { 165 | AGGREGATE(intValue, Int); 166 | } else { 167 | AGGREGATE(floatValue, Float); 168 | } 169 | #undef AGGREGATE 170 | } 171 | } 172 | // Not accepting aggregated columns before finalizing. 173 | return {false, true}; 174 | } 175 | 176 | columns = newColumns; 177 | return {true, true}; 178 | } 179 | 180 | bool SelectFilter::finalize(Columns &columns) { 181 | if (!isAggregated) { 182 | return false; 183 | } 184 | columns.resize(selectors.size()); 185 | for (int i = 0; i < selectors.size(); i++) { 186 | const auto &context = selectContexts[i]; 187 | const auto &selector = selectors[i]; 188 | columns[i].isNull = context.isNull; 189 | 190 | if (!context.isNull) { 191 | DataType dataType = selectIndexes[i] == -1 192 | ? INT 193 | : table->columns[selectIndexes[i]].type; 194 | #define CALC_AGGREGATION(_type, _Type) \ 195 | switch (selector.type) { \ 196 | case QuerySelector::AVG: \ 197 | columns[i].data.floatValue = \ 198 | float(context.value._type) / context.count; \ 199 | columns[i].type = FLOAT; \ 200 | break; \ 201 | case QuerySelector::MIN: \ 202 | case QuerySelector::MAX: \ 203 | case QuerySelector::SUM: \ 204 | columns[i].data._type = context.value._type; \ 205 | columns[i].type = _Type; \ 206 | break; \ 207 | case QuerySelector::COUNT_COL: \ 208 | case QuerySelector::COUNT_STAR: \ 209 | columns[i].data.intValue = context.value.intValue; \ 210 | columns[i].type = INT; \ 211 | break; \ 212 | default: \ 213 | assert(false); \ 214 | } 215 | if (dataType == INT) { 216 | CALC_AGGREGATION(intValue, INT); 217 | } else { 218 | CALC_AGGREGATION(floatValue, FLOAT); 219 | } 220 | #undef CALC_AGGREGATION 221 | } else { /* context.isNull */ 222 | if (selector.type == QuerySelector::COUNT_COL || 223 | selector.type == QuerySelector::COUNT_STAR) { 224 | columns[i].isNull = false; 225 | columns[i].data.intValue = 0; 226 | } 227 | } 228 | } 229 | 230 | return true; 231 | } 232 | // ====== End SelectFilter ====== 233 | 234 | // ===== Begin NullConditionFilter ===== 235 | void NullConditionFilter::build() { 236 | columnIndex = table->getColumnIndex(condition.columnId); 237 | if (columnIndex < 0) { 238 | throw Internal::ColumnNotFoundError(condition.columnId.getDesc()); 239 | } 240 | } 241 | 242 | std::pair NullConditionFilter::apply(Columns &columns) { 243 | const Column &column = columns[columnIndex]; 244 | 245 | bool accept = (condition.isNull && column.isNull) || 246 | (!condition.isNull && !column.isNull); 247 | return {accept, true}; 248 | } 249 | // ====== End NullConditionFilter ====== 250 | 251 | // ===== Begin ValueConditionFilter ===== 252 | // static bool _regexComparer(CompareOp op, const Column &column, 253 | // const char *regexStr) { 254 | // #ifdef DEBUG 255 | // if (op != LIKE) { 256 | // Logger::log(ERROR, 257 | // "RecordScanner: internal error: invalid compare op %d 258 | // " "for LIKE comparision\n", op); 259 | // throw Internal::UnexpedtedOperatorError(); 260 | // } 261 | // #endif 262 | // try { 263 | // std::regex regex(regexStr); 264 | // return std::regex_match(column.data.stringValue, regex); 265 | // } catch (std::regex_error &e) { 266 | // Logger::log(ERROR, "RecordScanner: invalid input regex \"%s\"\n", 267 | // regexStr); 268 | // throw Internal::InvalidRegexError(); 269 | // } 270 | // } 271 | 272 | using _Comparer = bool (*)(CompareOp, const char *, const char *); 273 | 274 | template 275 | static bool _comparer(CompareOp op, const char *lhs, const char *rhs) { 276 | T l = T(lhs); 277 | T r = T(rhs); 278 | switch (op) { 279 | case EQ: 280 | return l == r; 281 | case NE: 282 | return l != r; 283 | case LT: 284 | return l < r; 285 | case LE: 286 | return l <= r; 287 | case GT: 288 | return l > r; 289 | case GE: 290 | return l >= r; 291 | default: 292 | Logger::log(ERROR, 293 | "RecordScanner: internal error: invalid compare op %d " 294 | "for _comparer\n", 295 | op); 296 | throw Internal::UnexpedtedOperatorError(); 297 | } 298 | } 299 | 300 | void ValueConditionFilter::build() { 301 | columnIndex = table->getColumnIndex(condition.columnId); 302 | if (columnIndex < 0) { 303 | throw Internal::ColumnNotFoundError(condition.columnId.getDesc()); 304 | } 305 | } 306 | 307 | static _Comparer _getComparer(DataType type) { 308 | switch (type) { 309 | case DataType::INT: 310 | return _comparer; 311 | case DataType::FLOAT: 312 | return _comparer; 313 | case DataType::VARCHAR: 314 | return _comparer; 315 | default: 316 | assert(false); 317 | } 318 | } 319 | 320 | std::pair ValueConditionFilter::apply(Columns &columns) { 321 | const Column &column = columns[columnIndex]; 322 | 323 | // FIXME: What's the specification to deal with this? 324 | if (column.isNull) { 325 | return {false, true}; 326 | } 327 | 328 | _Comparer comparer = _getComparer(column.type); 329 | 330 | return {comparer(condition.op, column.data.stringValue, 331 | condition.value.stringValue), 332 | true}; 333 | } 334 | // ====== End ValueConditionFilter ====== 335 | 336 | // ===== Begin ColumnConditionFilter ===== 337 | void ColumnConditionFilter::build() { 338 | columnIndex1 = table->getColumnIndex(condition.lhs); 339 | columnIndex2 = table->getColumnIndex(condition.rhs); 340 | if (columnIndex1 < 0) { 341 | throw Internal::ColumnNotFoundError(condition.lhs.getDesc()); 342 | } 343 | if (columnIndex2 < 0) { 344 | throw Internal::ColumnNotFoundError(condition.rhs.getDesc()); 345 | } 346 | } 347 | 348 | std::pair ColumnConditionFilter::apply(Columns &columns) { 349 | const Column &column1 = columns[columnIndex1]; 350 | const Column &column2 = columns[columnIndex2]; 351 | 352 | // FIXME: Is this necessary? 353 | assert(column1.type == column2.type); 354 | 355 | _Comparer comparer = _getComparer(column1.type); 356 | 357 | return {comparer(condition.op, column1.data.stringValue, 358 | column2.data.stringValue), 359 | true}; 360 | } 361 | 362 | std::string QuerySelector::getColumnName() const { 363 | std::string desc = column.getDesc(); 364 | switch (type) { 365 | case COLUMN: 366 | return desc; 367 | case COUNT_STAR: 368 | return "COUNT(*)"; 369 | case COUNT_COL: 370 | return "COUNT(" + desc + ")"; 371 | case SUM: 372 | return "SUM(" + desc + ")"; 373 | case AVG: 374 | return "AVG(" + desc + ")"; 375 | case MIN: 376 | return "MIN(" + desc + ")"; 377 | case MAX: 378 | return "MAX(" + desc + ")"; 379 | } 380 | } 381 | 382 | } // namespace Internal 383 | } // namespace SimpleDB 384 | -------------------------------------------------------------------------------- /SimpleDB/src/dbms/SystemTableSchema.cc: -------------------------------------------------------------------------------- 1 | #include "DBMS.h" 2 | 3 | using namespace SimpleDB::Internal; 4 | 5 | namespace SimpleDB { 6 | std::vector DBMS::systemDatabaseTableColumns = { 7 | {.type = VARCHAR, 8 | .size = MAX_DATABASE_NAME_LEN, 9 | .nullable = false, 10 | .name = "name", 11 | .hasDefault = false}}; 12 | 13 | std::vector DBMS::systemTablesTableColumns = { 14 | {.type = VARCHAR, 15 | .size = MAX_TABLE_NAME_LEN, 16 | .nullable = false, 17 | .name = "name", 18 | .hasDefault = false}, 19 | {.type = VARCHAR, 20 | .size = MAX_DATABASE_NAME_LEN, 21 | .nullable = false, 22 | .name = "database", 23 | .hasDefault = false}, 24 | {.type = INT, 25 | .nullable = true, 26 | .name = "primary_key", 27 | .hasDefault = false}}; 28 | 29 | std::vector DBMS::systemIndexesTableColumns = { 30 | {.type = VARCHAR, 31 | .size = MAX_DATABASE_NAME_LEN, 32 | .nullable = false, 33 | .name = "database", 34 | .hasDefault = false}, 35 | {.type = VARCHAR, 36 | .size = MAX_TABLE_NAME_LEN, 37 | .nullable = false, 38 | .name = "table", 39 | .hasDefault = false}, 40 | {.type = VARCHAR, 41 | .size = MAX_COLUMN_NAME_LEN, 42 | .nullable = false, 43 | .name = "field", 44 | .hasDefault = false}, 45 | /* type 0: primary key index; type 1: ordinary index; type 2: migrated from 46 | ordinary index to primary key index */ 47 | {.type = INT, .nullable = false, .name = "type", .hasDefault = false}}; 48 | 49 | std::vector DBMS::systemForeignKeyTableColumns = { 50 | {.type = VARCHAR, 51 | .size = MAX_DATABASE_NAME_LEN, 52 | .nullable = false, 53 | .name = "database", 54 | .hasDefault = false}, 55 | {.type = VARCHAR, 56 | .size = MAX_TABLE_NAME_LEN, 57 | .nullable = false, 58 | .name = "table", 59 | .hasDefault = false}, 60 | {.type = VARCHAR, 61 | .size = MAX_COLUMN_NAME_LEN, 62 | .nullable = false, 63 | .name = "column", 64 | .hasDefault = false}, 65 | {.type = VARCHAR, 66 | .size = MAX_TABLE_NAME_LEN, 67 | .nullable = false, 68 | .name = "ref_table", 69 | .hasDefault = false}, 70 | {.type = VARCHAR, 71 | .size = MAX_COLUMN_NAME_LEN, 72 | .nullable = false, 73 | .name = "ref_column", 74 | .hasDefault = false}, 75 | }; 76 | 77 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/io/CacheManager.cc: -------------------------------------------------------------------------------- 1 | #include "internal/CacheManager.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include "internal/Logger.h" 8 | #include "internal/PageHandle.h" 9 | 10 | namespace SimpleDB { 11 | namespace Internal { 12 | 13 | CacheManager::CacheManager(FileManager *fileManager) { 14 | this->fileManager = fileManager; 15 | cacheBuf = new PageCache[NUM_BUFFER_PAGE]; 16 | activeCacheMapVec.resize(FileManager::MAX_OPEN_FILES); 17 | for (int i = 0; i < NUM_BUFFER_PAGE; i++) { 18 | cacheBuf[i].id = i; 19 | freeCache.insertHead(&cacheBuf[i]); 20 | } 21 | } 22 | 23 | CacheManager::~CacheManager() { close(); } 24 | 25 | void CacheManager::onCloseFile(FileDescriptor fd) { 26 | if (!fileManager->validate(fd)) { 27 | Logger::log( 28 | ERROR, 29 | "CacheManager: fail on closing file: invalid file descriptor: %d", 30 | fd.value); 31 | throw Internal::InvalidDescriptorError(); 32 | } 33 | 34 | auto &map = activeCacheMapVec[fd]; 35 | 36 | // Store the reference to avoid iterator invalidation. 37 | std::vector caches; 38 | for (auto &pair : map) { 39 | caches.push_back(pair.second); 40 | } 41 | 42 | for (auto cache : caches) { 43 | writeBack(cache); 44 | } 45 | } 46 | 47 | void CacheManager::close() { 48 | if (closed) { 49 | return; 50 | } 51 | 52 | std::vector caches; 53 | 54 | for (auto &map : activeCacheMapVec) { 55 | for (auto &pair : map) { 56 | caches.push_back(pair.second); 57 | } 58 | } 59 | 60 | for (auto cache : caches) { 61 | writeBack(cache); 62 | } 63 | 64 | closed = true; 65 | delete[] cacheBuf; 66 | } 67 | 68 | CacheManager::PageCache *CacheManager::getPageCache(FileDescriptor fd, 69 | int page) { 70 | if (!fileManager->validate(fd)) { 71 | Logger::log(ERROR, 72 | "CacheManager: fail to get page cache: invalid file " 73 | "descriptor: %d\n", 74 | fd.value); 75 | 76 | throw Internal::InvalidDescriptorError(); 77 | } 78 | 79 | if (page < 0) { 80 | Logger::log( 81 | ERROR, 82 | "CacheManager: fail to get page cache: invalid page number %d\n", 83 | page); 84 | throw Internal::InvalidPageNumberError(); 85 | } 86 | 87 | // Check if the page is in the cache. 88 | // TODO: Optimization. 89 | auto &cacheMap = activeCacheMapVec[fd]; 90 | auto iter = cacheMap.find(page); 91 | if (iter != cacheMap.end()) { 92 | // The page is cached. 93 | Logger::log(VERBOSE, "CacheManager: get cached page %d of file %d\n", 94 | page, fd.value); 95 | PageCache *cache = iter->second; 96 | 97 | // Move the cache to the head. 98 | activeCache.remove(cache->nodeInActiveCache); 99 | cache->nodeInActiveCache = activeCache.insertHead(cache); 100 | 101 | return cache; 102 | } 103 | 104 | // The page is not cached. 105 | PageCache *cache; 106 | 107 | if (freeCache.size() > 0) { 108 | // The cache is not full. 109 | Logger::log(VERBOSE, 110 | "CacheManager: get free cache for page %d of file %d\n", 111 | page, fd.value); 112 | 113 | cache = freeCache.removeTail(); 114 | assert(cache != nullptr); 115 | } else { 116 | // The cache is full. Select the last recently used page to replace. 117 | PageCache *lastCache = activeCache.last(); 118 | assert(lastCache != nullptr); 119 | 120 | Logger::log(VERBOSE, 121 | "CacheManager: replace cache of page %d of file %d for " 122 | "page %d of file %d\n", 123 | lastCache->meta.page, lastCache->meta.fd.value, page, 124 | fd.value); 125 | 126 | // Write back the original cache (the freed cache will be in the 127 | // `freeCache` list). 128 | writeBack(lastCache); 129 | cache = freeCache.removeTail(); 130 | } 131 | 132 | // Now we can claim this cache slot. 133 | cache->reset({fd, page}); 134 | 135 | // Read the page from disk as it is not cached. Note that the page might not 136 | // exist yet, so we must tolerate the error. 137 | fileManager->readPage(fd, page, cache->buf, true); 138 | 139 | // Now add this active cache to the map and the linked list. 140 | cacheMap[page] = cache; 141 | cache->nodeInActiveCache = activeCache.insertHead(cache); 142 | 143 | return cache; 144 | } 145 | 146 | PageHandle CacheManager::getHandle(FileDescriptor fd, int page) { 147 | PageCache *cache = getPageCache(fd, page); 148 | 149 | return PageHandle(cache); 150 | } 151 | 152 | PageHandle CacheManager::renew(const PageHandle &handle) { 153 | if (handle.validate()) { 154 | return handle; 155 | } 156 | 157 | return getHandle(handle.cache->meta.fd, handle.cache->meta.page); 158 | } 159 | 160 | char *CacheManager::load(const PageHandle &handle) { 161 | if (!handle.validate()) { 162 | Logger::log(DEBUG_, 163 | "CacheManager: trying to read data with an outdated page " 164 | "handle for page %d of file %d\n", 165 | handle.cache->meta.page, handle.cache->meta.fd.value); 166 | return nullptr; 167 | } 168 | 169 | return handle.cache->buf; 170 | } 171 | 172 | char *CacheManager::loadRaw(const PageHandle &handle) { 173 | #ifdef DEBUG 174 | if (!handle.validate()) { 175 | Logger::log(ERROR, 176 | "CacheManager: trying to read data with an outdated page " 177 | "handle for page %d of file %d\n", 178 | handle.cache->meta.page, handle.cache->meta.fd.value); 179 | assert(false); 180 | throw Internal::InvalidPageHandleError(); 181 | } 182 | #endif 183 | return handle.cache->buf; 184 | } 185 | 186 | void CacheManager::markDirty(const PageHandle &handle) { 187 | PageCache *cache = handle.cache; 188 | 189 | if (!handle.validate()) { 190 | Logger::log( 191 | ERROR, 192 | "CacheManager: fail to modify page %d of file %d: " 193 | "possible outdated page handle: current generation %d, got %d\n", 194 | cache->meta.fd.value, cache->meta.page, cache->generation, 195 | handle.generation); 196 | throw Internal::InvalidPageHandleError(); 197 | } 198 | 199 | cache->dirty = true; 200 | } 201 | 202 | void CacheManager::writeBack(PageCache *cache) { 203 | // As we are dealing with a valid pointer to the cache, we assume that the 204 | // descriptor is valid. 205 | 206 | if (cache->dirty) { 207 | Logger::log(VERBOSE, 208 | "CacheManager: write back dirty page %d of file %d\n", 209 | cache->meta.page, cache->meta.fd.value); 210 | fileManager->writePage(cache->meta.fd, cache->meta.page, cache->buf); 211 | } else { 212 | Logger::log(VERBOSE, "CacheManager: discarding page %d of file %d\n", 213 | cache->meta.page, cache->meta.fd.value); 214 | } 215 | 216 | // Discard the cache and add it back to the free list. 217 | activeCacheMapVec[cache->meta.fd].erase(cache->meta.page); 218 | activeCache.remove(cache->nodeInActiveCache); 219 | freeCache.insertHead(cache); 220 | // Don't forget to bump the generation number, as the previous cache is no 221 | // longer valid. 222 | cache->generation++; 223 | } 224 | 225 | void CacheManager::writeBack(const PageHandle &handle) { 226 | PageCache *cache = handle.cache; 227 | 228 | if (!handle.validate()) { 229 | Logger::log( 230 | ERROR, 231 | "CacheManager: fail to write back page %d of file %d: " 232 | "possible outdated page handle: current generation %d, got %d\n", 233 | cache->meta.fd.value, cache->meta.page, cache->generation, 234 | handle.generation); 235 | throw Internal::InvalidPageHandleError(); 236 | } 237 | 238 | writeBack(cache); 239 | } 240 | 241 | #if TESTING 242 | // ==== Testing-only methods ==== 243 | void CacheManager::discard(FileDescriptor fd, int page) { 244 | auto &cacheMap = activeCacheMapVec[fd]; 245 | auto iter = cacheMap.find(page); 246 | if (iter != cacheMap.end()) { 247 | PageCache *cache = iter->second; 248 | activeCacheMapVec[cache->meta.fd].erase(cache->meta.page); 249 | activeCache.remove(cache->nodeInActiveCache); 250 | freeCache.insertHead(cache); 251 | cache->generation++; 252 | } 253 | } 254 | 255 | void CacheManager::discardAll(FileDescriptor fd) { 256 | auto &map = activeCacheMapVec[fd]; 257 | std::vector caches; 258 | for (auto &pair : map) { 259 | caches.push_back(pair.second); 260 | } 261 | for (auto cache : caches) { 262 | discard(cache->meta.fd, cache->meta.page); 263 | } 264 | } 265 | 266 | void CacheManager::discardAll() { 267 | std::vector caches; 268 | for (auto &map : activeCacheMapVec) { 269 | for (auto &pair : map) { 270 | caches.push_back(pair.second); 271 | } 272 | } 273 | for (auto cache : caches) { 274 | discard(cache->meta.fd, cache->meta.page); 275 | } 276 | } 277 | #endif 278 | 279 | } // namespace Internal 280 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/io/FileCoordinator.cc: -------------------------------------------------------------------------------- 1 | #include "internal/FileCoordinator.h" 2 | 3 | #include "internal/PageHandle.h" 4 | 5 | namespace SimpleDB { 6 | namespace Internal { 7 | 8 | FileCoordinator FileCoordinator::shared = FileCoordinator(); 9 | 10 | FileCoordinator::FileCoordinator() { 11 | fileManager = new FileManager(); 12 | cacheManager = new CacheManager(fileManager); 13 | } 14 | 15 | FileCoordinator::~FileCoordinator() { 16 | delete cacheManager; 17 | delete fileManager; 18 | } 19 | 20 | void FileCoordinator::createFile(const std::string &fileName) { 21 | fileManager->createFile(fileName); 22 | } 23 | 24 | FileDescriptor FileCoordinator::openFile(const std::string &fileName) { 25 | return fileManager->openFile(fileName); 26 | } 27 | 28 | void FileCoordinator::closeFile(FileDescriptor fd) { 29 | cacheManager->onCloseFile(fd); 30 | fileManager->closeFile(fd); 31 | } 32 | 33 | void FileCoordinator::deleteFile(const std::string &fileName) { 34 | fileManager->deleteFile(fileName); 35 | } 36 | 37 | PageHandle FileCoordinator::getHandle(FileDescriptor fd, int page) { 38 | return cacheManager->getHandle(fd, page); 39 | } 40 | 41 | char *FileCoordinator::load(PageHandle *handle) { 42 | *handle = cacheManager->renew(*handle); 43 | return cacheManager->loadRaw(*handle); 44 | } 45 | 46 | void FileCoordinator::markDirty(const PageHandle &handle) { 47 | cacheManager->markDirty(handle); 48 | } 49 | 50 | PageHandle FileCoordinator::renew(const PageHandle &handle) { 51 | return cacheManager->renew(handle); 52 | } 53 | 54 | } // namespace Internal 55 | } -------------------------------------------------------------------------------- /SimpleDB/src/io/FileManager.cc: -------------------------------------------------------------------------------- 1 | #include "internal/FileManager.h" 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "internal/Logger.h" 9 | 10 | namespace SimpleDB { 11 | namespace Internal { 12 | 13 | void FileManager::createFile(const std::string &fileName) { 14 | if (std::filesystem::exists(fileName)) { 15 | Logger::log(ERROR, "FileManager: file %s already exists\n", 16 | fileName.c_str()); 17 | throw Internal::FileExistsError(); 18 | } 19 | 20 | FILE *fd = fopen(fileName.c_str(), "a"); 21 | 22 | if (fd == nullptr) { 23 | Logger::log(ERROR, "FileManager: failed to create file %s\n", 24 | fileName.c_str()); 25 | throw Internal::CreateFileError(); 26 | } 27 | 28 | Logger::log(VERBOSE, "FileManager: created file %s\n", fileName.c_str()); 29 | fclose(fd); 30 | } 31 | 32 | FileDescriptor FileManager::openFile(const std::string &fileName) { 33 | FILE *fd = fopen(fileName.c_str(), "rb+"); 34 | 35 | if (fd == nullptr) { 36 | Logger::log(ERROR, "FileManager: failed to open file %s\n", 37 | fileName.c_str()); 38 | throw Internal::OpenFileError(); 39 | } 40 | 41 | Logger::log(VERBOSE, "FileManager: opened file %s\n", fileName.c_str()); 42 | return genNewDescriptor(fd, fileName); 43 | } 44 | 45 | void FileManager::closeFile(FileDescriptor descriptor) { 46 | if (!validate(descriptor)) { 47 | Logger::log(ERROR, 48 | "FileManager: fail to close file: invalid descriptor %d\n", 49 | descriptor.value); 50 | throw Internal::InvalidDescriptorError(); 51 | } 52 | 53 | OpenedFile &file = openedFiles[descriptor]; 54 | int err = fclose(file.fd); 55 | file.fd = nullptr; 56 | descriptorBitmap &= ~(1 << descriptor); 57 | 58 | if (err) { 59 | Logger::log(ERROR, "FileManager: fail to close file %s: %s\n", 60 | file.fileName.c_str(), strerror(errno)); 61 | throw Internal::CloseFileError(); 62 | } 63 | 64 | Logger::log(VERBOSE, "FileManager: closed file %s\n", 65 | file.fileName.c_str()); 66 | } 67 | 68 | void FileManager::deleteFile(const std::string &fileName) { 69 | int err = remove(fileName.c_str()); 70 | 71 | if (err) { 72 | Logger::log(ERROR, "FileManager: fail to delete file %s: %s\n", 73 | fileName.c_str(), strerror(errno)); 74 | throw Internal::DeleteFileError(); 75 | } 76 | 77 | Logger::log(VERBOSE, "FileManager: deleted file %s\n", fileName.c_str()); 78 | } 79 | 80 | void FileManager::readPage(FileDescriptor descriptor, int page, char *data, 81 | bool couldFail) { 82 | if (!validate(descriptor)) { 83 | Logger::log(ERROR, 84 | "FileManager: fail to read page: invalid descriptor %d\n", 85 | descriptor.value); 86 | throw Internal::InvalidDescriptorError(); 87 | } 88 | 89 | if (page < 0) { 90 | Logger::log(ERROR, 91 | "FileManager: fail to read page: invalid page number %d\n", 92 | page); 93 | throw Internal::InvalidPageNumberError(); 94 | } 95 | 96 | const OpenedFile &file = openedFiles[descriptor]; 97 | FILE *fd = file.fd; 98 | 99 | int err = fseek(fd, page * PAGE_SIZE, SEEK_SET); 100 | if (err) { 101 | if (!couldFail) { 102 | Logger::log( 103 | ERROR, 104 | "FileManager: fail to read page of file %s: seek to page " 105 | "%d failed: %s\n", 106 | file.fileName.c_str(), page, strerror(errno)); 107 | throw Internal::ReadFileError(); 108 | } else { 109 | return; 110 | } 111 | } 112 | 113 | size_t readSize = fread(data, 1, PAGE_SIZE, fd); 114 | if (readSize != PAGE_SIZE) { 115 | if (!couldFail) { 116 | Logger::log( 117 | ERROR, 118 | "FileManager: fail to read page of file %s: read page %d " 119 | "failed (read size %lu)\n", 120 | file.fileName.c_str(), page, readSize); 121 | throw Internal::ReadFileError(); 122 | } else { 123 | return; 124 | } 125 | } 126 | 127 | Logger::log(VERBOSE, "FileManager: read page %d from file %s\n", page, 128 | file.fileName.c_str()); 129 | } 130 | 131 | void FileManager::writePage(FileDescriptor descriptor, int page, char *data) { 132 | if (!validate(descriptor)) { 133 | Logger::log(ERROR, 134 | "FileManager: fail to write page: invalid descriptor %d\n", 135 | descriptor.value); 136 | throw Internal::InvalidDescriptorError(); 137 | } 138 | 139 | if (page < 0) { 140 | Logger::log(ERROR, 141 | "FileManager: fail to write page: invalid page number %d\n", 142 | page); 143 | throw Internal::InvalidPageNumberError(); 144 | } 145 | 146 | const OpenedFile &file = openedFiles[descriptor]; 147 | FILE *fd = file.fd; 148 | 149 | int err = fseek(fd, page * PAGE_SIZE, SEEK_SET); 150 | if (err) { 151 | Logger::log(ERROR, 152 | "FileManager: fail to write page of file %s: seek to page " 153 | "%d failed: %s\n", 154 | file.fileName.c_str(), page, strerror(errno)); 155 | throw Internal::WriteFileError(); 156 | } 157 | 158 | size_t writeSize = fwrite(data, 1, PAGE_SIZE, fd); 159 | if (writeSize != PAGE_SIZE) { 160 | Logger::log( 161 | ERROR, 162 | "FileManager: fail to write page %d of file %s (write size: %lu)\n", 163 | page, file.fileName.c_str(), writeSize); 164 | throw Internal::WriteFileError(); 165 | } 166 | 167 | Logger::log(VERBOSE, "FileManager: wrote page %d to file %s\n", page, 168 | file.fileName.c_str()); 169 | } 170 | 171 | bool FileManager::validate(FileDescriptor fd) { 172 | return fd >= 0 && fd < MAX_OPEN_FILES && 173 | (descriptorBitmap & (1L << fd)) != 0; 174 | } 175 | 176 | FileDescriptor FileManager::genNewDescriptor(FILE *fd, 177 | const std::string &fileName) { 178 | // Find the first unset bit. 179 | int index = ffsll(~descriptorBitmap); 180 | 181 | if (index == 0) { 182 | Logger::log(ERROR, "FileManager: Number of opened files exceeded.\n"); 183 | throw Internal::OpenFileExceededError(); 184 | } 185 | 186 | // ffsll returns a 1-based index. 187 | index--; 188 | 189 | openedFiles[index] = {fileName, fd}; 190 | descriptorBitmap |= (1L << index); 191 | return FileDescriptor(index); 192 | } 193 | 194 | } // namespace Internal 195 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDB/src/util/Logger.cc: -------------------------------------------------------------------------------- 1 | #include "internal/Logger.h" 2 | 3 | #include 4 | #include 5 | 6 | namespace SimpleDB { 7 | namespace Internal { 8 | 9 | LogLevel Logger::displayMinLevel = NOTICE; 10 | FILE *Logger::errorStream = stderr; 11 | 12 | void Logger::log(LogLevel level, const char *fmt, ...) { 13 | if (level < displayMinLevel) return; 14 | fprintf(errorStream, "[%-7s] ", logLevelNames[level]); 15 | 16 | va_list argList; 17 | 18 | va_start(argList, fmt); 19 | vfprintf(errorStream, fmt, argList); 20 | va_end(argList); 21 | } 22 | 23 | } // namespace Internal 24 | } // namespace SimpleDB -------------------------------------------------------------------------------- /SimpleDBClient/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .DS_Store 132 | .vscode 133 | 134 | # PyCharm or other JetBrains IDEs 135 | .idea 136 | -------------------------------------------------------------------------------- /SimpleDBClient/BUILD: -------------------------------------------------------------------------------- 1 | load("@simpledb_service_dep//:requirements.bzl", "requirement") 2 | 3 | py_binary( 4 | name = "main", 5 | srcs = glob(["*.py"]), 6 | deps = [ 7 | "//:simpledb_service_py", 8 | requirement("prompt_toolkit"), 9 | requirement("Pygments"), 10 | requirement("prettytable"), 11 | ], 12 | visibility = ["//visibility:public"] 13 | ) 14 | -------------------------------------------------------------------------------- /SimpleDBClient/client.py: -------------------------------------------------------------------------------- 1 | import SimpleDBService.query_pb2_grpc as service 2 | import grpc 3 | from SimpleDBService.query_pb2 import ExecutionRequest 4 | 5 | class SimpleDBClient(service.QueryServicer): 6 | def __init__(self, addr: str) -> None: 7 | super().__init__() 8 | self.addr = addr 9 | self.channel = None 10 | 11 | def connect(self): 12 | self.channel = grpc.insecure_channel(self.addr) 13 | self.stub = service.QueryStub(self.channel) 14 | 15 | def execute(self, sql: str): 16 | request = ExecutionRequest() 17 | request.sql = sql 18 | return self.ExecuteSQLProgram(request, None) 19 | 20 | def ExecuteSQLProgram(self, request, context): 21 | return self.stub.ExecuteSQLProgram(request) 22 | -------------------------------------------------------------------------------- /SimpleDBClient/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import csv 4 | from prompt_toolkit import prompt 5 | from pygments.lexers.sql import SqlLexer 6 | from prompt_toolkit.lexers import PygmentsLexer 7 | import SimpleDBService.query_pb2 as query_pb2 8 | import grpc 9 | 10 | from util import print_html, print_bold, print_table, print_duration, transform_sql_literal 11 | from client import SimpleDBClient 12 | 13 | lexer = PygmentsLexer(SqlLexer) 14 | 15 | STARTUP_PROMPT = """\ 16 | SimpleDB Client 17 | By: Liang Yesheng <liang2kl@outlook.com> 18 | """ 19 | 20 | USAGE = """\ 21 | Usage 22 | Execute: Esc + Enter 23 | Discard: Ctrl + C 24 | Terminate: Ctrl + D 25 | """ 26 | 27 | client: SimpleDBClient = None 28 | current_db = "" 29 | 30 | def connect_server(addr: str): 31 | global client 32 | try: 33 | client = SimpleDBClient(addr) 34 | client.connect() 35 | except: 36 | return False 37 | return True 38 | 39 | def value_desc(value) -> str: 40 | if value.HasField("int_value"): 41 | return str(value.int_value) 42 | elif value.HasField("float_value"): 43 | return str(value.float_value) 44 | elif value.HasField("varchar_value"): 45 | return value.varchar_value 46 | elif value.HasField("null_value"): 47 | return "NULL" 48 | 49 | def print_resp(resp): 50 | if (resp.HasField("error")): 51 | err_name = query_pb2.ExecutionError.Type.Name(resp.error.type) 52 | print_bold(f"ERROR", end="") 53 | print_bold(f"({err_name}):", resp.error.message) 54 | elif resp.result.HasField("plain"): 55 | print(resp.result.plain.msg, end = "") 56 | if resp.result.plain.affected_rows >= 0: 57 | print(f", {resp.result.plain.affected_rows} rows affected") 58 | else: 59 | print("") 60 | elif resp.result.HasField("show_databases"): 61 | print_table(["Database"], 62 | [[x] for x in resp.result.show_databases.databases]) 63 | elif resp.result.HasField("show_table"): 64 | print_table(["Table"], 65 | [[x] for x in resp.result.show_table.tables]) 66 | elif resp.result.HasField("describe_table"): 67 | print_table(["Field", "Type", "Null", "Key", "Default"], 68 | [[x.field, x.type, "YES" if x.nullable else "NO", "PRI" if x.primary_key else "", 69 | x.default_value] for x in resp.result.describe_table.columns]) 70 | elif resp.result.HasField("show_indexes"): 71 | print_table(["Table", "Column", "Key Name"], 72 | [[x.table, x.column, "PRIMARY" if x.is_pk else x.column] 73 | for x in resp.result.show_indexes.indexes]) 74 | elif resp.result.HasField("query"): 75 | num = len(resp.result.query.rows) 76 | print_num = num 77 | if num > 100: 78 | try: 79 | text = prompt(f"Too many rows ({num}), print? [y/N/] (default: N) ") 80 | except EOFError: 81 | return 82 | except KeyboardInterrupt: 83 | return 84 | try: 85 | print_num = int(text) 86 | if (print_num <= 0): 87 | return 88 | print_num = print_num if print_num < num else num 89 | print(f"Showing the first {print_num} rows") 90 | except ValueError: 91 | if text != "y": 92 | return 93 | print_table([x.name for x in resp.result.query.columns], 94 | [[value_desc(x) for x in row.values] for row in resp.result.query.rows[:print_num]], 95 | true_num=num) 96 | else: 97 | print(resp.result) 98 | 99 | def send_request(sql: str): 100 | try: 101 | batch_resp = client.execute(sql) 102 | responses = batch_resp.responses 103 | except grpc.RpcError as e: 104 | print(f"RPC Error ({e.code()}):", e.details(), end="\n\n") 105 | return 106 | 107 | if len(responses) == 1: 108 | print_resp(responses[0]) 109 | else: 110 | for i, resp in enumerate(responses): 111 | print_bold(f"Result [{i}]") 112 | print_resp(resp) 113 | 114 | print_duration(batch_resp.stats.elapse) 115 | print("") 116 | 117 | if len(responses) > 0: 118 | global current_db 119 | current_db = responses[-1].current_db 120 | 121 | def send_request_silent(sql: str): 122 | try: 123 | batch_resp = client.execute(sql) 124 | responses = batch_resp.responses 125 | except grpc.RpcError as e: 126 | return f"RPC Error ({e.code()}): " + e.details() 127 | 128 | if len(responses) != 1: 129 | return "Multiple responses" 130 | resp = responses[0] 131 | if resp.HasField("error"): 132 | err_name = query_pb2.ExecutionError.Type.Name(resp.error.type) 133 | return f"ERROR ({err_name}): " + resp.error.message 134 | return None 135 | 136 | def main_loop(): 137 | while True: 138 | try: 139 | text = prompt(">>> " + (f"({current_db}) " if current_db else ""), 140 | multiline=True, lexer=lexer) 141 | send_request(text) 142 | except KeyboardInterrupt: 143 | continue 144 | except EOFError: 145 | break 146 | except BaseException as e: 147 | print("Exception occured:", e) 148 | break 149 | 150 | 151 | if __name__ == "__main__": 152 | parser = argparse.ArgumentParser() 153 | parser.add_argument("--server", "-s", action="store", dest="server_addr", 154 | default="127.0.0.0:9100", 155 | help="Listening address of SimpleDB gRPC server") 156 | parser.add_argument("--csv", action="store", dest="csv_file", 157 | help="CSV file to import") 158 | parser.add_argument("--db", "-d", action="store", dest="db_name", help="Database name to import") 159 | parser.add_argument("--table", "-t", action="store", dest="table_name", help="Table name to import") 160 | 161 | result = parser.parse_args(sys.argv[1:]) 162 | 163 | print_html(STARTUP_PROMPT) 164 | print_bold("Run configurations") 165 | for k, v in result.__dict__.items(): 166 | if v is not None: 167 | print(f" {k.replace('_', ' ').capitalize()}: {v}") 168 | print("") 169 | 170 | success = connect_server(result.server_addr) 171 | 172 | if not success: 173 | print("Error: failed to connect to server, exiting...") 174 | sys.exit(1) 175 | 176 | if result.csv_file is not None: 177 | print(f"Importing {result.csv_file} to {result.db_name}.{result.table_name}...") 178 | err = send_request_silent("USE " + result.db_name + ";") 179 | if err is not None: 180 | print(err) 181 | exit(1) 182 | with open(result.csv_file, "r") as f: 183 | reader = csv.reader(f) 184 | lines = 0 185 | for row in reader: 186 | row = [transform_sql_literal(r) for r in row] 187 | sql = f"INSERT INTO {result.table_name} VALUES ({','.join(row)});" 188 | err = send_request_silent(sql) 189 | if err is not None: 190 | print(err + " (" + sql + ")") 191 | exit(1) 192 | else: 193 | lines += 1 194 | print("Processed %d rows" % lines, end="\r") 195 | else: 196 | print_html(USAGE) 197 | main_loop() 198 | 199 | -------------------------------------------------------------------------------- /SimpleDBClient/requirements.txt: -------------------------------------------------------------------------------- 1 | prompt-toolkit==3.0.0 2 | Pygments==2.13.0 3 | wcwidth==0.2.5 4 | prettytable==3.5.0 5 | -------------------------------------------------------------------------------- /SimpleDBClient/util.py: -------------------------------------------------------------------------------- 1 | import pygments 2 | from prettytable import PrettyTable 3 | from prompt_toolkit import print_formatted_text 4 | from prompt_toolkit import HTML, ANSI 5 | 6 | # print = print_formatted_text 7 | 8 | class Colors: 9 | """ ANSI color codes """ 10 | BLACK = "\033[0;30m" 11 | RED = "\033[0;31m" 12 | GREEN = "\033[0;32m" 13 | BROWN = "\033[0;33m" 14 | BLUE = "\033[0;34m" 15 | PURPLE = "\033[0;35m" 16 | CYAN = "\033[0;36m" 17 | LIGHT_GRAY = "\033[0;37m" 18 | DARK_GRAY = "\033[1;30m" 19 | LIGHT_RED = "\033[1;31m" 20 | LIGHT_GREEN = "\033[1;32m" 21 | YELLOW = "\033[1;33m" 22 | LIGHT_BLUE = "\033[1;34m" 23 | LIGHT_PURPLE = "\033[1;35m" 24 | LIGHT_CYAN = "\033[1;36m" 25 | LIGHT_WHITE = "\033[1;37m" 26 | BOLD = "\033[1m" 27 | FAINT = "\033[2m" 28 | ITALIC = "\033[3m" 29 | UNDERLINE = "\033[4m" 30 | BLINK = "\033[5m" 31 | NEGATIVE = "\033[7m" 32 | CROSSED = "\033[9m" 33 | END = "\033[0m" 34 | 35 | def print_html(s, *args, **kwargs): 36 | print_formatted_text(HTML(s), *args, **kwargs) 37 | 38 | def print_bold(s, *args, **kwargs): 39 | print_html(f"{s}", *args, **kwargs) 40 | 41 | def print_table(headers, rows, true_num=-1): 42 | if len(rows) == 0: 43 | print("Empty set") 44 | return 45 | table = PrettyTable() 46 | # prettytable will raise "Field names must be unique" on duplicate column 47 | # names, but we just need that. 48 | table._validate_field_names = lambda *a, **k: None 49 | table.field_names = headers 50 | table.align = "l" 51 | for r in rows: 52 | table.add_row(r) 53 | print(table) 54 | print(f"{len(rows) if true_num < 0 else true_num} rows in set") 55 | 56 | def print_duration(duration): 57 | if (duration > 1e6): 58 | desc = f"{duration / 1e6:.3f}s" 59 | elif (duration > 1e3): 60 | desc = f"{duration / 1e3:.3f}ms" 61 | else: 62 | desc = f"{duration:.3f}us" 63 | print(Colors.FAINT + desc + Colors.END) 64 | 65 | def transform_sql_literal(val: str): 66 | try: 67 | float(val) 68 | return val 69 | except ValueError: 70 | return "'" + val + "'" -------------------------------------------------------------------------------- /SimpleDBServer/BUILD: -------------------------------------------------------------------------------- 1 | cc_binary( 2 | name = "main", 3 | deps = [ 4 | "//:simpledb", 5 | "@com_github_gflags_gflags//:gflags" 6 | ], 7 | srcs = glob(["src/**"]), 8 | visibility = ["//visibility:public"], 9 | linkstatic = True, 10 | ) 11 | -------------------------------------------------------------------------------- /SimpleDBServer/src/SQLService.cc: -------------------------------------------------------------------------------- 1 | #include "SQLService.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | using grpc::ServerContext; 17 | using grpc::Status; 18 | using grpc::StatusCode; 19 | using SimpleDB::Service::ExecutionBatchResponse; 20 | using SimpleDB::Service::ExecutionRequest; 21 | using SimpleDB::Service::ExecutionResponse; 22 | using SimpleDB::Service::ExecutionResult; 23 | using ErrorType = SimpleDB::Service::ExecutionError::Type; 24 | 25 | using namespace SimpleDB::Error; 26 | 27 | #define STOP_CLOCK() \ 28 | end = steady_clock::now(); \ 29 | duration = duration_cast(end - begin).count(); \ 30 | response->mutable_stats()->set_elapse(duration); 31 | 32 | SQLService::SQLService(SimpleDB::DBMS* dbms) : dbms(dbms) {} 33 | 34 | Status SQLService::ExecuteSQLProgram(ServerContext* context, 35 | const ExecutionRequest* request, 36 | ExecutionBatchResponse* response) { 37 | // Lock the mutex to prevent concurrent execution. 38 | std::lock_guard _(mutex); 39 | 40 | const std::string sql = request->sql(); 41 | std::istringstream stream(sql); 42 | 43 | using std::chrono::duration_cast; 44 | using std::chrono::steady_clock; 45 | 46 | steady_clock::time_point begin = steady_clock::now(); 47 | steady_clock::time_point end; 48 | long long duration; 49 | 50 | try { 51 | std::vector results = dbms->executeSQL(stream); 52 | 53 | STOP_CLOCK(); 54 | 55 | for (const auto& result : results) { 56 | ExecutionResponse* resp = response->add_responses(); 57 | *resp->mutable_current_db() = dbms->getCurrentDatabase(); 58 | resp->mutable_result()->CopyFrom(result); 59 | } 60 | 61 | return Status::OK; 62 | } catch (SyntaxError& e) { 63 | makeError(response, ErrorType::ExecutionError_Type_ERR_SYNTAX, 64 | e.what()); 65 | STOP_CLOCK(); 66 | return Status::OK; 67 | } catch (IncompatableValueError& e) { 68 | makeError(response, 69 | ErrorType::ExecutionError_Type_ERR_INCOMPATIBLE_VALUE, 70 | e.what()); 71 | STOP_CLOCK(); 72 | return Status::OK; 73 | } catch (DatabaseExistsError& e) { 74 | makeError(response, ErrorType::ExecutionError_Type_ERR_DATABASE_EXIST, 75 | e.what()); 76 | STOP_CLOCK(); 77 | return Status::OK; 78 | } catch (CreateDatabaseError& e) { 79 | makeError(response, ErrorType::ExecutionError_Type_ERR_CREATE_DATABASE, 80 | e.what()); 81 | STOP_CLOCK(); 82 | return Status::OK; 83 | } catch (DatabaseNotExistError& e) { 84 | makeError(response, 85 | ErrorType::ExecutionError_Type_ERR_DATABASE_NOT_EXIST, 86 | e.what()); 87 | STOP_CLOCK(); 88 | return Status::OK; 89 | } catch (UninitializedError& e) { 90 | makeError(response, ErrorType::ExecutionError_Type_ERR_UNINITIALIZED, 91 | e.what()); 92 | STOP_CLOCK(); 93 | return Status::OK; 94 | } catch (InitializationError& e) { 95 | makeError(response, ErrorType::ExecutionError_Type_ERR_INITIALIZATION, 96 | e.what()); 97 | STOP_CLOCK(); 98 | return Status::OK; 99 | } catch (InvalidDatabaseNameError& e) { 100 | makeError(response, ErrorType::ExecutionError_Type_ERR_INVAL_DB_NAME, 101 | e.what()); 102 | STOP_CLOCK(); 103 | return Status::OK; 104 | } catch (DatabaseNotSelectedError& e) { 105 | makeError(response, ErrorType::ExecutionError_Type_ERR_DB_NOT_SELECTED, 106 | e.what()); 107 | STOP_CLOCK(); 108 | return Status::OK; 109 | } catch (TableExistsError& e) { 110 | makeError(response, ErrorType::ExecutionError_Type_ERR_TABLE_EXISTS, 111 | e.what()); 112 | STOP_CLOCK(); 113 | return Status::OK; 114 | } catch (InvalidTableNameError& e) { 115 | makeError(response, ErrorType::ExecutionError_Type_ERR_INVAL_TABLE_NAME, 116 | e.what()); 117 | STOP_CLOCK(); 118 | return Status::OK; 119 | } catch (TableNotExistsError& e) { 120 | makeError(response, ErrorType::ExecutionError_Type_ERR_TABLE_NOT_EXISTS, 121 | e.what()); 122 | STOP_CLOCK(); 123 | return Status::OK; 124 | } catch (MultiplePrimaryKeyError& e) { 125 | makeError(response, ErrorType::ExecutionError_Type_ERR_MUL_PRIKEY, 126 | e.what()); 127 | STOP_CLOCK(); 128 | return Status::OK; 129 | } catch (CreateTableError& e) { 130 | makeError(response, ErrorType::ExecutionError_Type_ERR_CREATE_TABLE, 131 | e.what()); 132 | STOP_CLOCK(); 133 | return Status::OK; 134 | } catch (AlterPrimaryKeyError& e) { 135 | makeError(response, ErrorType::ExecutionError_Type_ERR_ALT_PRIKEY, 136 | e.what()); 137 | STOP_CLOCK(); 138 | return Status::OK; 139 | } catch (AlterForeignKeyError& e) { 140 | makeError(response, ErrorType::ExecutionError_Type_ERR_ALT_FOREIGN_KEY, 141 | e.what()); 142 | STOP_CLOCK(); 143 | return Status::OK; 144 | } catch (AlterIndexError& e) { 145 | makeError(response, ErrorType::ExecutionError_Type_ERR_ALT_INDEX, 146 | e.what()); 147 | STOP_CLOCK(); 148 | return Status::OK; 149 | } catch (InsertError& e) { 150 | makeError(response, ErrorType::ExecutionError_Type_ERR_INSERT, 151 | e.what()); 152 | STOP_CLOCK(); 153 | return Status::OK; 154 | } catch (SelectError& e) { 155 | makeError(response, ErrorType::ExecutionError_Type_ERR_SELECT, 156 | e.what()); 157 | STOP_CLOCK(); 158 | return Status::OK; 159 | } catch (UpdateError& e) { 160 | makeError(response, ErrorType::ExecutionError_Type_ERR_UPDATE, 161 | e.what()); 162 | STOP_CLOCK(); 163 | return Status::OK; 164 | } catch (DeleteError& e) { 165 | makeError(response, ErrorType::ExecutionError_Type_ERR_DELETE, 166 | e.what()); 167 | STOP_CLOCK(); 168 | return Status::OK; 169 | } catch (InternalError& e) { 170 | makeError(response, ErrorType::ExecutionError_Type_ERR_INTERNAL, 171 | e.what()); 172 | STOP_CLOCK(); 173 | return Status::OK; 174 | } catch (std::exception& e) { 175 | STOP_CLOCK(); 176 | return Status(StatusCode::INTERNAL, 177 | std::string("Unexpected exception occured: ") + e.what()); 178 | } 179 | } 180 | 181 | void SQLService::makeError(ExecutionBatchResponse* response, ErrorType type, 182 | const std::string& message) { 183 | ExecutionResponse resp; 184 | resp.mutable_error()->set_type(type); 185 | resp.mutable_error()->set_message(message); 186 | response->mutable_responses()->Add(std::move(resp)); 187 | } -------------------------------------------------------------------------------- /SimpleDBServer/src/SQLService.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | class SQLService final : public SimpleDB::Service::Query::Service { 10 | public: 11 | SQLService(SimpleDB::DBMS* dbms); 12 | virtual ::grpc::Status ExecuteSQLProgram( 13 | ::grpc::ServerContext* context, 14 | const ::SimpleDB::Service::ExecutionRequest* request, 15 | ::SimpleDB::Service::ExecutionBatchResponse* response) override; 16 | 17 | private: 18 | SimpleDB::DBMS* dbms; 19 | void makeError(SimpleDB::Service::ExecutionBatchResponse* resp, 20 | SimpleDB::Service::ExecutionError::Type type, 21 | const std::string& message); 22 | std::mutex mutex; 23 | }; 24 | -------------------------------------------------------------------------------- /SimpleDBServer/src/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "SQLService.h" 25 | 26 | // Forward declarations 27 | void runServer(); 28 | void sigintHandler(int); 29 | void initFromCLIFlags(int argc, char *argv[]); 30 | void checkShutdown(); 31 | 32 | std::mutex mutex; 33 | std::condition_variable cv; 34 | 35 | // CLI flags and validators 36 | DEFINE_string(dir, "", "Root directory of the database data"); 37 | DEFINE_bool(verbose, false, "Output verbose logs"); 38 | DEFINE_bool(debug, false, "Output debug logs"); 39 | DEFINE_bool(silent, false, "Silence all logs"); 40 | DEFINE_string(addr, "127.0.0.1:9100", "Server bind address"); 41 | DEFINE_string(log, "", "Log file path"); 42 | DEFINE_validator(dir, [](const char *flagName, const std::string &value) { 43 | if (value.empty()) { 44 | std::cerr << "ERROR: --" << flagName << " must be specified" 45 | << std::endl; 46 | return false; 47 | } 48 | return true; 49 | }); 50 | 51 | SimpleDB::DBMS *dbms; 52 | std::shared_ptr server; 53 | 54 | int main(int argc, char *argv[]) { 55 | // Parse CLI flags 56 | initFromCLIFlags(argc, argv); 57 | 58 | // Register handlers 59 | signal(SIGINT, sigintHandler); 60 | 61 | try { 62 | dbms->init(); 63 | } catch (SimpleDB::Error::InitializationError &e) { 64 | std::cerr << e.what() << std::endl; 65 | delete dbms; 66 | return -1; 67 | } 68 | 69 | // Start grpc server. 70 | runServer(); 71 | 72 | return 0; 73 | } 74 | 75 | void initFromCLIFlags(int argc, char *argv[]) { 76 | gflags::ParseCommandLineFlags(&argc, &argv, false); 77 | 78 | SimpleDB::Internal::LogLevel logLevel = 79 | SimpleDB::Internal::LogLevel::NOTICE; 80 | 81 | if (FLAGS_silent) { 82 | logLevel = SimpleDB::Internal::LogLevel::SILENT; 83 | } else if (FLAGS_debug) { 84 | logLevel = SimpleDB::Internal::LogLevel::DEBUG_; 85 | } else if (FLAGS_verbose) { 86 | logLevel = SimpleDB::Internal::LogLevel::VERBOSE; 87 | } 88 | 89 | SimpleDB::Internal::Logger::setLogLevel(logLevel); 90 | 91 | if (!FLAGS_log.empty()) { 92 | FILE *file = fopen(FLAGS_log.c_str(), "w"); 93 | if (file == nullptr) { 94 | std::cerr << "ERROR: Failed to open log file: " << FLAGS_log 95 | << std::endl; 96 | exit(-1); 97 | } 98 | SimpleDB::Internal::Logger::setErrorStream(file); 99 | } 100 | 101 | dbms = new SimpleDB::DBMS(FLAGS_dir); 102 | } 103 | 104 | void runServer() { 105 | SQLService service(dbms); 106 | grpc::ServerBuilder builder; 107 | grpc::ResourceQuota quota; 108 | 109 | std::cout << "Server listening on " << FLAGS_addr << std::endl; 110 | 111 | builder.AddListeningPort(FLAGS_addr, grpc::InsecureServerCredentials()) 112 | .RegisterService(&service); 113 | server = builder.BuildAndStart(); 114 | 115 | // Create a shutdown thread. 116 | std::thread t(checkShutdown); 117 | t.join(); 118 | 119 | server->Wait(); 120 | 121 | delete dbms; 122 | } 123 | 124 | void sigintHandler(int) { 125 | // Exit gracefully... 126 | std::cout << "\nShutting down server..." << std::endl; 127 | cv.notify_one(); 128 | } 129 | 130 | void checkShutdown() { 131 | std::unique_lock lock(mutex); 132 | cv.wait(lock); 133 | server->Shutdown(); 134 | } 135 | -------------------------------------------------------------------------------- /SimpleDBService/BUILD: -------------------------------------------------------------------------------- 1 | load("@rules_proto_grpc//cpp:defs.bzl", "cpp_grpc_library") 2 | load("@rules_proto_grpc//python:defs.bzl", "python_grpc_library") 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | proto_library( 7 | name = "query_proto", 8 | srcs = ["query.proto"], 9 | ) 10 | 11 | cpp_grpc_library( 12 | name = "simpledb_service", 13 | protos = [":query_proto"], 14 | linkstatic = True, 15 | ) 16 | 17 | python_grpc_library( 18 | name = "simpledb_service_py", 19 | protos = [":query_proto"], 20 | ) 21 | -------------------------------------------------------------------------------- /SimpleDBService/query.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package SimpleDB.Service; 3 | 4 | service Query { 5 | rpc ExecuteSQLProgram (ExecutionRequest) returns (ExecutionBatchResponse); 6 | } 7 | 8 | message ExecutionRequest { 9 | string sql = 1; 10 | } 11 | 12 | message ExecutionBatchResponse { 13 | repeated ExecutionResponse responses = 1; 14 | Stats stats = 4; 15 | } 16 | 17 | message ExecutionResponse { 18 | oneof response { 19 | ExecutionResult result = 1; 20 | ExecutionError error = 2; 21 | } 22 | string current_db = 3; 23 | } 24 | 25 | message Stats { 26 | int32 elapse = 1; 27 | } 28 | 29 | message ExecutionResult { 30 | oneof data { 31 | PlainResult plain = 1; 32 | QueryResult query = 2; 33 | ShowTableResult show_table = 3; 34 | ShowDatabasesResult show_databases = 4; 35 | DescribeTableResult describe_table = 5; 36 | ShowIndexesResult show_indexes = 6; 37 | } 38 | } 39 | 40 | message ExecutionError { 41 | enum Type { 42 | ERR_UNSPECIFIED = 0; 43 | ERR_INTERNAL = 1; 44 | ERR_UNKNOWN = 2; 45 | ERR_INCOMPATIBLE_VALUE = 3; 46 | ERR_SYNTAX = 4; 47 | ERR_DATABASE_EXIST = 5; 48 | ERR_CREATE_DATABASE = 6; 49 | ERR_DATABASE_NOT_EXIST = 7; 50 | ERR_UNINITIALIZED = 8; 51 | ERR_INITIALIZATION = 9; 52 | ERR_INVAL_DB_NAME = 10; 53 | ERR_DB_NOT_SELECTED = 11; 54 | ERR_TABLE_EXISTS = 12; 55 | ERR_INVAL_TABLE_NAME = 13; 56 | ERR_TABLE_NOT_EXISTS = 14; 57 | ERR_MUL_PRIKEY = 15; 58 | ERR_CREATE_TABLE = 16; 59 | ERR_ALT_PRIKEY = 17; 60 | ERR_ALT_FOREIGN_KEY = 18; 61 | ERR_ALT_INDEX = 19; 62 | ERR_INSERT = 20; 63 | ERR_SELECT = 21; 64 | ERR_UPDATE = 22; 65 | ERR_DELETE = 23; 66 | } 67 | string message = 1; 68 | Type type = 2; 69 | } 70 | 71 | message PlainResult { 72 | string msg = 1; 73 | int32 affected_rows = 2; 74 | } 75 | 76 | // Query result: result for queries. 77 | message QueryResult { 78 | repeated QueryColumn columns = 1; 79 | repeated QueryRow rows = 2; 80 | } 81 | 82 | message QueryColumn { 83 | enum Type { 84 | TYPE_UNSPECIFIED = 0; 85 | TYPE_INT = 1; 86 | TYPE_FLOAT = 2; 87 | TYPE_VARCHAR = 3; 88 | } 89 | string name = 1; 90 | Type type = 2; 91 | } 92 | 93 | message QueryRow { 94 | repeated QueryValue values = 1; 95 | } 96 | 97 | message QueryValue { 98 | oneof value { 99 | int32 int_value = 1; 100 | float float_value = 2; 101 | string varchar_value = 3; 102 | bool null_value = 4; 103 | } 104 | } 105 | 106 | // SHOW TABLE result. 107 | message ShowTableResult { 108 | repeated string tables = 1; 109 | } 110 | 111 | // SHOW DATABASAES result. 112 | message ShowDatabasesResult { 113 | repeated string databases = 1; 114 | } 115 | 116 | // DESCRIBE TABLE result. 117 | message DescribeTableResult { 118 | repeated ColumnDescription columns = 1; 119 | } 120 | 121 | message ColumnDescription { 122 | string field = 1; 123 | string type = 2; 124 | bool nullable = 3; 125 | bool primary_key = 4; 126 | // TODO: Primary key / foreign key 127 | optional string default_value = 5; 128 | } 129 | 130 | message ShowIndexesResult { 131 | repeated IndexDescription indexes = 1; 132 | } 133 | 134 | message IndexDescription { 135 | string table = 1; 136 | string column = 2; 137 | bool is_pk = 3; 138 | } 139 | -------------------------------------------------------------------------------- /SimpleDBTest/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | tmp/ -------------------------------------------------------------------------------- /SimpleDBTest/BUILD: -------------------------------------------------------------------------------- 1 | load("@rules_cc//cc:defs.bzl", "cc_test") 2 | 3 | cc_test( 4 | name = "simpledb_test", 5 | size = "small", 6 | srcs = glob(["*.cc", "*.h"]), 7 | copts = ["-std=c++17"], 8 | deps = [ 9 | "@googletest//:gtest", 10 | "//:simpledb-test", 11 | ], 12 | local_defines = ["DEBUG=1", "TESTING=1"], 13 | visibility = ["//visibility:public"], 14 | linkstatic = True, 15 | ) 16 | -------------------------------------------------------------------------------- /SimpleDBTest/CacheManagerTest.cc: -------------------------------------------------------------------------------- 1 | #ifndef TESTING 2 | #define TESTING 1 3 | #endif 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include "Util.h" 11 | 12 | using namespace SimpleDB; 13 | using namespace SimpleDB::Internal; 14 | 15 | #ifdef PAGE_SIZE 16 | #undef PAGE_SIZE 17 | #endif 18 | 19 | class CacheManagerTest : public ::testing::Test { 20 | protected: 21 | CacheManagerTest() { 22 | fileManager = new FileManager(); 23 | manager = new CacheManager(fileManager); 24 | } 25 | 26 | ~CacheManagerTest() { 27 | delete manager; 28 | delete fileManager; 29 | } 30 | 31 | void SetUp() override { std::filesystem::create_directory("tmp"); } 32 | void TearDown() override { std::filesystem::remove_all("tmp"); } 33 | 34 | FileManager *fileManager; 35 | CacheManager *manager; 36 | }; 37 | 38 | TEST_F(CacheManagerTest, TestReadWritePage) { 39 | const char filePath[] = "tmp/file"; 40 | char readBuf[PAGE_SIZE]; 41 | char buf[PAGE_SIZE]; 42 | 43 | for (int i = 0; i < PAGE_SIZE; i++) { 44 | buf[i] = char(rand()); 45 | } 46 | 47 | fileManager->createFile(filePath); 48 | FileDescriptor fd = fileManager->openFile(filePath); 49 | fileManager->writePage(fd, 1, buf); 50 | 51 | PageHandle handle; 52 | 53 | // Read page. 54 | EXPECT_NO_THROW(handle = manager->getHandle(fd, 1)); 55 | EXPECT_TRUE(handle.validate()); 56 | EXPECT_EQ(manager->load(handle), handle.cache->buf); 57 | 58 | // Write cache. 59 | char *cacheBuf = manager->load(handle); 60 | char c = cacheBuf[0]; 61 | while (c == cacheBuf[0]) { 62 | c = char(rand()); 63 | } 64 | cacheBuf[0] = c; 65 | 66 | EXPECT_FALSE(handle.cache->dirty); 67 | EXPECT_NO_THROW(manager->markDirty(handle)); 68 | EXPECT_TRUE(handle.cache->dirty); 69 | 70 | EXPECT_EQ(handle.cache->buf[0], c); 71 | 72 | // Write back. 73 | EXPECT_NO_THROW(manager->writeBack(handle)); 74 | memset(readBuf, 0, PAGE_SIZE); 75 | fileManager->readPage(fd, 1, readBuf); 76 | EXPECT_EQ(memcmp(cacheBuf, readBuf, PAGE_SIZE), 0); 77 | 78 | // Cleanup. 79 | EXPECT_NO_THROW(manager->onCloseFile(fd)); 80 | fileManager->closeFile(fd); 81 | } 82 | 83 | TEST_F(CacheManagerTest, TestPageExchange) { 84 | DisableLogGuard guard; 85 | 86 | const char filePath[] = "tmp/file"; 87 | 88 | fileManager->createFile(filePath); 89 | FileDescriptor fd = fileManager->openFile(filePath); 90 | 91 | std::vector handles; 92 | 93 | for (int i = 0; i < NUM_BUFFER_PAGE; i++) { 94 | PageHandle handle; 95 | ASSERT_NO_THROW(handle = manager->getHandle(fd, i)); 96 | ASSERT_NO_THROW(manager->markDirty(handle)); 97 | handles.push_back(handle); 98 | } 99 | 100 | // Validate LRU algorithm. 101 | EXPECT_EQ(manager->activeCache.tail->next->data->id, 0); 102 | 103 | manager->getHandle(fd, 5); 104 | EXPECT_EQ(manager->activeCache.head->data->id, 5); 105 | EXPECT_EQ(manager->freeCache.size(), 0); 106 | 107 | manager->getHandle(fd, NUM_BUFFER_PAGE); 108 | EXPECT_EQ(manager->activeCache.tail->next->data->id, 1); 109 | EXPECT_EQ(manager->freeCache.size(), 0); 110 | 111 | // At this time, the cache of page 0 should be written back, thus 112 | // invalidating the handle. 113 | EXPECT_FALSE(handles[0].validate()); 114 | 115 | EXPECT_NO_THROW(manager->discardAll(fd)); 116 | fileManager->closeFile(fd); 117 | } 118 | 119 | TEST_F(CacheManagerTest, TestLeak) { 120 | DisableLogGuard guard; 121 | 122 | const char filePath[] = "tmp/file"; 123 | 124 | fileManager->createFile(filePath); 125 | FileDescriptor fd = fileManager->openFile(filePath); 126 | 127 | for (int i = 0; i < 20; i++) { 128 | PageHandle handle; 129 | ASSERT_NO_THROW(handle = manager->getHandle(fd, i)); 130 | EXPECT_EQ(manager->freeCache.size(), NUM_BUFFER_PAGE - i - 1); 131 | EXPECT_EQ(manager->activeCache.size(), i + 1); 132 | } 133 | 134 | EXPECT_NO_THROW(manager->onCloseFile(fd)); 135 | EXPECT_EQ(manager->freeCache.size(), NUM_BUFFER_PAGE); 136 | EXPECT_EQ(manager->activeCache.size(), 0); 137 | 138 | fileManager->closeFile(fd); 139 | } 140 | -------------------------------------------------------------------------------- /SimpleDBTest/FileCoordinatorTest.cc: -------------------------------------------------------------------------------- 1 | #ifndef TESTING 2 | #define TESTING 1 3 | #endif 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | using namespace SimpleDB; 11 | using namespace SimpleDB::Internal; 12 | 13 | #ifdef PAGE_SIZE 14 | #undef PAGE_SIZE 15 | #endif 16 | 17 | class FileCoordinatorTest : public ::testing::Test { 18 | protected: 19 | void SetUp() override { std::filesystem::create_directory("tmp"); } 20 | void TearDown() override { std::filesystem::remove_all("tmp"); } 21 | 22 | FileCoordinator coordinator; 23 | }; 24 | 25 | // The same as FileManagerTest.TestWriteReadPage 26 | TEST_F(FileCoordinatorTest, TestCoordinator) { 27 | // Initialize data. 28 | char buf[PAGE_SIZE] = {0x00, 0x00, 0x12, 0x24}; 29 | 30 | const char filePath[] = "tmp/file-rw"; 31 | EXPECT_NO_THROW(coordinator.createFile(filePath)); 32 | ASSERT_TRUE(std::filesystem::exists(filePath)) 33 | << "Fail to create file " << filePath; 34 | 35 | FileDescriptor fd = coordinator.openFile(filePath); 36 | PageHandle handle = coordinator.getHandle(fd, 2); 37 | 38 | memcpy(coordinator.load(&handle), buf, PAGE_SIZE); 39 | ASSERT_NO_THROW(coordinator.markDirty(handle)); 40 | 41 | ASSERT_NO_THROW(coordinator.closeFile(fd)); 42 | ASSERT_NO_THROW(fd = coordinator.openFile(filePath)); 43 | ASSERT_NO_THROW(handle = coordinator.getHandle(fd, 2)); 44 | 45 | EXPECT_EQ(memcmp(buf, coordinator.load(&handle), PAGE_SIZE), 0) 46 | << "Read data mismatch with written data"; 47 | 48 | coordinator.closeFile(fd); 49 | 50 | // The CacheManager should have cleared the cache of this file. 51 | EXPECT_EQ(coordinator.cacheManager->activeCache.size(), 0); 52 | // The FileManager should have released the file descriptor. 53 | EXPECT_EQ(coordinator.fileManager->descriptorBitmap, 0); 54 | } 55 | 56 | TEST_F(FileCoordinatorTest, TestRenewHandle) { 57 | const char filePath[] = "tmp/file-rw"; 58 | ASSERT_NO_THROW(coordinator.createFile(filePath)); 59 | ASSERT_TRUE(std::filesystem::exists(filePath)) 60 | << "Fail to create file " << filePath; 61 | 62 | FileDescriptor fd = coordinator.openFile(filePath); 63 | PageHandle handle = coordinator.getHandle(fd, 2); 64 | 65 | EXPECT_NE(coordinator.load(&handle), nullptr); 66 | coordinator.cacheManager->writeBack(handle.cache); 67 | 68 | EXPECT_EQ(coordinator.cacheManager->load(handle), nullptr); 69 | // Renew happens here. 70 | EXPECT_NE(coordinator.load(&handle), nullptr); 71 | EXPECT_NE(coordinator.cacheManager->load(handle), nullptr); 72 | } 73 | -------------------------------------------------------------------------------- /SimpleDBTest/FileManagerTest.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "Util.h" 8 | 9 | #ifdef PAGE_SIZE 10 | #undef PAGE_SIZE 11 | #endif 12 | 13 | using namespace SimpleDB; 14 | using namespace SimpleDB::Internal; 15 | 16 | class FileManagerTest : public ::testing::Test { 17 | protected: 18 | void SetUp() override { std::filesystem::create_directory("tmp"); } 19 | void TearDown() override { std::filesystem::remove_all("tmp"); } 20 | 21 | FileManager manager; 22 | }; 23 | 24 | TEST_F(FileManagerTest, TestCreateRemoveFile) { 25 | DisableLogGuard guard; 26 | 27 | for (int i = 0; i < FileManager::MAX_OPEN_FILES; i++) { 28 | char filePath[50]; 29 | snprintf(filePath, 10, "tmp/file-%d", i); 30 | // Remove files before testing. 31 | std::filesystem::remove(filePath); 32 | 33 | ASSERT_NO_THROW(manager.createFile(filePath)); 34 | EXPECT_TRUE(std::filesystem::exists(filePath)) 35 | << "Fail to create file " << filePath; 36 | 37 | FileDescriptor fd = manager.openFile(filePath); 38 | 39 | ASSERT_NO_THROW(manager.closeFile(fd)); 40 | ASSERT_NO_THROW(manager.deleteFile(filePath)); 41 | EXPECT_FALSE(std::filesystem::exists(filePath)) 42 | << "Fail to delete file " << filePath; 43 | ; 44 | } 45 | } 46 | 47 | TEST_F(FileManagerTest, TestWriteReadPage) { 48 | // Initialize data. 49 | char buf[PAGE_SIZE] = {0x00, 0x00, 0x12, 0x24}; 50 | buf[PAGE_SIZE - 2] = 0x36; 51 | 52 | const char filePath[] = "tmp/file-rw"; 53 | ASSERT_NO_THROW(manager.createFile(filePath)); 54 | ASSERT_TRUE(std::filesystem::exists(filePath)) 55 | << "Fail to create file " << filePath; 56 | 57 | FileDescriptor fd = manager.openFile(filePath); 58 | 59 | ASSERT_NO_THROW(manager.writePage(fd, 2, buf)); 60 | 61 | char readBuf[PAGE_SIZE]; 62 | ASSERT_NO_THROW(manager.readPage(fd, 2, readBuf)); 63 | 64 | EXPECT_EQ(memcmp(buf, readBuf, PAGE_SIZE), 0) 65 | << "Read data mismatch with written data"; 66 | EXPECT_EQ(readBuf[PAGE_SIZE - 2], 0x36) 67 | << "Read data mismatch with written data"; 68 | 69 | manager.closeFile(fd); 70 | } 71 | 72 | TEST_F(FileManagerTest, TestExceedFiles) { 73 | DisableLogGuard guard; 74 | 75 | for (int i = 0; i < FileManager::MAX_OPEN_FILES; i++) { 76 | char filePath[50]; 77 | snprintf(filePath, 10, "tmp/file-%d", i); 78 | std::filesystem::remove(filePath); 79 | 80 | manager.createFile(filePath); 81 | manager.openFile(filePath); 82 | } 83 | 84 | char filePath[] = "tmp/file-overflow"; 85 | 86 | ASSERT_NO_THROW(manager.createFile(filePath)); 87 | EXPECT_THROW(manager.openFile(filePath), Internal::OpenFileExceededError) 88 | << "Open files exceeded but not thrown"; 89 | ASSERT_NO_THROW(manager.deleteFile(filePath)); 90 | } 91 | 92 | TEST_F(FileManagerTest, TestInvalidFileDescriptor) { 93 | DisableLogGuard guard; 94 | 95 | char buf[PAGE_SIZE]; 96 | 97 | for (int fdValue = -1; fdValue <= FileManager::MAX_OPEN_FILES; fdValue++) { 98 | FileDescriptor fd(fdValue); 99 | EXPECT_THROW(manager.closeFile(fd), Internal::InvalidDescriptorError) 100 | << "Close file with invalid descriptor but not thrown"; 101 | EXPECT_THROW(manager.readPage(fd, 0, buf), 102 | Internal::InvalidDescriptorError) 103 | << "Read page with invalid descriptor but not thrown"; 104 | EXPECT_THROW(manager.writePage(fd, 0, buf), 105 | Internal::InvalidDescriptorError) 106 | << "Write page with invalid descriptor but not thrown"; 107 | } 108 | } 109 | 110 | TEST_F(FileManagerTest, TestInvalidPageNumber) { 111 | const char filePath[] = "tmp/file"; 112 | ASSERT_NO_THROW(manager.createFile(filePath)); 113 | 114 | FileDescriptor fd; 115 | ASSERT_NO_THROW(fd = manager.openFile(filePath)); 116 | 117 | char buf[PAGE_SIZE]; 118 | 119 | EXPECT_THROW(manager.readPage(fd, -1, buf), 120 | Internal::InvalidPageNumberError); 121 | ASSERT_NO_THROW(manager.deleteFile(filePath)); 122 | } 123 | -------------------------------------------------------------------------------- /SimpleDBTest/IndexTest.cc: -------------------------------------------------------------------------------- 1 | #ifndef TESTING 2 | #define TESTING 1 3 | #endif 4 | #ifndef DEBUG 5 | #define DEBUG 1 6 | #endif 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include "Util.h" 15 | 16 | using namespace SimpleDB; 17 | using namespace SimpleDB::Internal; 18 | 19 | class IndexTest : public ::testing::Test { 20 | protected: 21 | void SetUp() override { 22 | std::filesystem::create_directory("tmp"); 23 | srand(0); 24 | } 25 | void TearDown() override { 26 | index.close(); 27 | std::filesystem::remove_all("tmp"); 28 | } 29 | 30 | Index index; 31 | 32 | const char *indexFile = "tmp/index"; 33 | 34 | void initIndex() { ASSERT_NO_THROW(index.create(indexFile)); } 35 | 36 | void reloadIndex() { 37 | index.close(); 38 | ASSERT_NO_THROW(index.open(indexFile)); 39 | } 40 | }; 41 | 42 | TEST_F(IndexTest, TestUninitializeAccess) { 43 | int key = 1; 44 | EXPECT_THROW(index.insert(key, false, {0, 0}), 45 | Internal::IndexNotInitializedError); 46 | EXPECT_THROW(index.remove(key, false, {0, 0}), 47 | Internal::IndexNotInitializedError); 48 | EXPECT_THROW(index.findEq(key, false), Internal::IndexNotInitializedError); 49 | } 50 | 51 | TEST_F(IndexTest, TestCreateNewIndex) { 52 | initIndex(); 53 | reloadIndex(); 54 | 55 | EXPECT_EQ(index.meta.numNode, 1); 56 | EXPECT_EQ(index.meta.rootNode, 0); 57 | } 58 | 59 | TEST_F(IndexTest, TestInitFromInvalidFile) { 60 | const char *fileName = "tmp/invalid_file"; 61 | PF::create(fileName); 62 | EXPECT_THROW(index.open(fileName), Internal::ReadIndexError); 63 | } 64 | 65 | TEST_F(IndexTest, TestInsertGet) { 66 | DisableLogGuard _; 67 | initIndex(); 68 | 69 | // Create a set of random entries. 70 | std::set keys; 71 | 72 | while (keys.size() < 1000 * MAX_NUM_ENTRY_PER_NODE) { 73 | keys.insert(rand()); 74 | } 75 | 76 | std::vector> entries; 77 | 78 | for (auto key : keys) { 79 | entries.push_back({key, {rand(), rand()}}); 80 | entries.push_back({key, {rand(), rand()}}); 81 | } 82 | 83 | std::shuffle(entries.begin(), entries.end(), std::mt19937(0)); 84 | 85 | for (int i = 0; i < entries.size(); i++) { 86 | auto [key, rid] = entries[i]; 87 | ASSERT_NO_THROW(index.insert(key, false, rid)); 88 | EXPECT_EQ(index.meta.numEntry, i + 1); 89 | } 90 | 91 | reloadIndex(); 92 | 93 | // Validate the entries. 94 | std::shuffle(entries.begin(), entries.end(), std::mt19937(0)); 95 | 96 | for (auto entry : entries) { 97 | int key = entry.first; 98 | std::vector rids; 99 | ASSERT_NO_THROW(rids = index.findEq(key, false)); 100 | ASSERT_EQ(rids.size(), 2); 101 | EXPECT_TRUE(rids[0] == entry.second || rids[1] == entry.second); 102 | } 103 | 104 | FileCoordinator::shared.cacheManager->discardAll(index.fd); 105 | } 106 | 107 | TEST_F(IndexTest, TestRemove) { 108 | DisableLogGuard _; 109 | initIndex(); 110 | 111 | // Create a set of random entries. 112 | std::set keys; 113 | 114 | while (keys.size() < 1000 * MAX_NUM_ENTRY_PER_NODE) { 115 | keys.insert(rand()); 116 | } 117 | 118 | std::vector> entries; 119 | 120 | for (auto key : keys) { 121 | entries.push_back({key, {rand(), rand()}}); 122 | } 123 | 124 | std::shuffle(entries.begin(), entries.end(), std::mt19937(0)); 125 | 126 | for (auto entry : entries) { 127 | int key = entry.first; 128 | RecordID rid = entry.second; 129 | ASSERT_NO_THROW(index.insert(key, false, rid)); 130 | } 131 | 132 | // Remove the entries. 133 | std::shuffle(entries.begin(), entries.end(), std::mt19937(0)); 134 | 135 | for (int i = 0; i < entries.size(); i++) { 136 | auto [key, rid] = entries[i]; 137 | 138 | ASSERT_NO_THROW(index.remove(key, false, rid)); 139 | EXPECT_EQ(index.meta.numEntry, entries.size() - i - 1); 140 | 141 | // Test remove non-exist key. 142 | ASSERT_THROW(index.remove(key, false, rid), 143 | Internal::IndexKeyNotExistsError); 144 | } 145 | 146 | FileCoordinator::shared.cacheManager->discardAll(index.fd); 147 | } -------------------------------------------------------------------------------- /SimpleDBTest/IndexedTableTest.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "Util.h" 13 | 14 | using namespace SimpleDB; 15 | using namespace SimpleDB::Internal; 16 | 17 | class IndexedTableTest : public ::testing::Test { 18 | protected: 19 | void SetUp() override { 20 | std::filesystem::create_directory("tmp"); 21 | table.create("test", "test", schema); 22 | index = std::make_shared(); 23 | index->create("index"); 24 | srand(0); 25 | } 26 | void TearDown() override { std::filesystem::remove_all("tmp"); } 27 | 28 | const std::string colName = "col"; 29 | const std::vector schema = { 30 | {.type = INT, .nullable = false, .name = "col"}, 31 | }; 32 | 33 | Table table; 34 | std::shared_ptr index; 35 | 36 | CompareValueCondition cond(CompareOp op, int i) { 37 | ColumnValue v; 38 | v.intValue = i; 39 | return CompareValueCondition({.columnName = colName}, op, v); 40 | } 41 | }; 42 | 43 | TEST_F(IndexedTableTest, TestRangeCollapse) { 44 | using TestCase = std::tuple, 45 | std::vector>; 46 | 47 | std::vector testCases = { 48 | {{cond(NE, 1), cond(GE, 0), cond(LE, 3)}, {{0, 0}, {2, 3}}}, 49 | {{cond(NE, 1), cond(EQ, 1)}, {}}, 50 | {{cond(GE, 0), cond(LE, 1), cond(NE, 0), cond(NE, 1)}, {}}, 51 | {{cond(GE, 0), cond(LE, 4), cond(NE, 1), cond(NE, 3)}, 52 | {{0, 0}, {2, 2}, {4, 4}}}, 53 | {{cond(NE, 0)}, {{INT_MIN, -1}, {1, INT_MAX}}}, 54 | {{cond(GE, 0), cond(NE, INT_MAX)}, {{0, INT_MAX - 1}}}, 55 | }; 56 | 57 | Table t; 58 | 59 | for (const auto &testCase : testCases) { 60 | auto &[conditions, expected] = testCase; 61 | IndexedTable t = IndexedTable( 62 | &table, 63 | [&](const std::string &, const std::string &) { return index; }); 64 | for (const auto &condition : conditions) { 65 | bool accepted = t.acceptCondition(condition); 66 | ASSERT_TRUE(accepted); 67 | } 68 | 69 | t.collapseRanges(); 70 | 71 | ASSERT_EQ(t.ranges.size(), expected.size()); 72 | for (size_t i = 0; i < t.ranges.size(); i++) { 73 | EXPECT_EQ(t.ranges[i].first, expected[i].first); 74 | EXPECT_EQ(t.ranges[i].second, expected[i].second); 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /SimpleDBTest/ParameterTest.cc: -------------------------------------------------------------------------------- 1 | #ifndef TESTING 2 | #define TESTING 1 3 | #endif 4 | #include 5 | #include 6 | 7 | using namespace SimpleDB; 8 | using namespace SimpleDB::Internal; 9 | -------------------------------------------------------------------------------- /SimpleDBTest/QueryConditionTest.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "Util.h" 9 | 10 | using namespace SimpleDB; 11 | using namespace SimpleDB::Internal; 12 | 13 | class QueryConditionTest : public ::testing::Test { 14 | protected: 15 | void SetUp() override { 16 | std::filesystem::create_directory("tmp"); 17 | initTable(); 18 | } 19 | void TearDown() override { 20 | table.close(); 21 | std::filesystem::remove_all("tmp"); 22 | } 23 | 24 | Table table; 25 | 26 | std::vector columnMetas = { 27 | {.type = INT, .size = 4, .nullable = true, .name = "int_val"}, 28 | {.type = FLOAT, .size = 4, .nullable = true, .name = "float_val"}, 29 | {.type = VARCHAR, .size = 100, .nullable = true, .name = "varchar_val"}, 30 | // A nullable column. 31 | {.type = INT, .size = 4, .nullable = true, .name = "int_val_nullable"}, 32 | }; 33 | 34 | const char *testVarChar = "Hello, world"; 35 | const char *tableName = "table_name"; 36 | 37 | void initTable() { 38 | ASSERT_NO_THROW(table.create("tmp/table", tableName, columnMetas)); 39 | } 40 | 41 | QueryBuilder getBaseBuilder() { return QueryBuilder(&table); } 42 | }; 43 | 44 | TEST_F(QueryConditionTest, TestCompareInt) { 45 | Columns testColumns0 = {Column(1), Column(1.1F), Column(testVarChar, 100), 46 | Column::nullIntColumn()}; 47 | Columns testColumns1 = {Column(5), Column(-1.1F), Column(testVarChar, 100), 48 | Column::nullIntColumn()}; 49 | Columns testColumns2 = {Column(3), Column(1.1F), Column(testVarChar, 100), 50 | Column::nullIntColumn()}; 51 | 52 | ASSERT_NO_THROW(table.insert(testColumns0)); 53 | ASSERT_NO_THROW(table.insert(testColumns1)); 54 | ASSERT_NO_THROW(table.insert(testColumns2)); 55 | 56 | QueryBuilder::Result result; 57 | 58 | // EQ. 59 | int value = 1; 60 | QueryBuilder builder = getBaseBuilder(); 61 | builder.condition(columnMetas[0].name, EQ, (const char *)(&value)); 62 | 63 | ASSERT_NO_THROW(result = builder.execute()); 64 | ASSERT_EQ(result.size(), 1); 65 | compareColumns(result[0].second, testColumns0); 66 | 67 | // GT. 68 | value = 4; 69 | builder = getBaseBuilder(); 70 | builder.condition(columnMetas[0].name, GT, (const char *)(&value)); 71 | ASSERT_NO_THROW(result = builder.execute()); 72 | ASSERT_EQ(result.size(), 1); 73 | compareColumns(result[0].second, testColumns1); 74 | 75 | // Emptyset. 76 | value = 10; 77 | builder = getBaseBuilder(); 78 | builder.condition(columnMetas[0].name, GE, (const char *)(&value)); 79 | ASSERT_NO_THROW(result = builder.execute()); 80 | ASSERT_EQ(result.size(), 0); 81 | 82 | // Two constraints (GE, LT). 83 | int value1 = 3, value2 = 5; 84 | builder = getBaseBuilder(); 85 | builder.condition(columnMetas[0].name, GE, (const char *)(&value1)); 86 | builder.condition(columnMetas[0].name, LT, (const char *)(&value2)); 87 | ASSERT_NO_THROW(result = builder.execute()); 88 | ASSERT_EQ(result.size(), 1); 89 | compareColumns(result[0].second, testColumns2); 90 | } 91 | 92 | TEST_F(QueryConditionTest, TestCompareFloat) { 93 | Columns testColumns0 = {Column(0), Column(1.1F), Column(testVarChar, 100), 94 | Column::nullIntColumn()}; 95 | Columns testColumns1 = {Column(0), Column(-1.1F), Column(testVarChar, 100), 96 | Column::nullIntColumn()}; 97 | 98 | ASSERT_NO_THROW(table.insert(testColumns0)); 99 | ASSERT_NO_THROW(table.insert(testColumns1)); 100 | 101 | // EQ and GE. 102 | std::vector> pairs = { 103 | std::make_tuple(EQ, atof("1.1"), testColumns0), 104 | std::make_tuple(GE, atof("1.1"), testColumns0), 105 | std::make_tuple(GE, atof("1.0"), testColumns0), 106 | std::make_tuple(EQ, atof("-1.1"), testColumns1), 107 | std::make_tuple(LE, atof("-1.1"), testColumns1), 108 | std::make_tuple(LE, atof("-1.0"), testColumns1), 109 | std::make_tuple(GT, atof("1.0"), testColumns0), 110 | std::make_tuple(LT, atof("-1.0"), testColumns1)}; 111 | 112 | for (auto &pair : pairs) { 113 | float value = std::get<1>(pair); 114 | const Columns &testColumn = std::get<2>(pair); 115 | 116 | QueryBuilder builder = getBaseBuilder(); 117 | builder.condition(columnMetas[1].name, std::get<0>(pair), 118 | (const char *)(&value)); 119 | 120 | QueryBuilder::Result result; 121 | ASSERT_NO_THROW(result = builder.execute()); 122 | ASSERT_EQ(result.size(), 1); 123 | compareColumns(result[0].second, testColumn); 124 | } 125 | } 126 | 127 | TEST_F(QueryConditionTest, TestCompareVarchar) { 128 | Columns testColumns0 = {Column(1), Column(1.1F), Column("aaabbb", 100), 129 | Column::nullIntColumn()}; 130 | Columns testColumns1 = {Column(5), Column(-1.0F), Column("b", 100), 131 | Column::nullIntColumn()}; 132 | Columns testColumns2 = {Column(1), Column(1.1F), Column("aaaaaa", 100), 133 | Column::nullIntColumn()}; 134 | 135 | ASSERT_NO_THROW(table.insert(testColumns0)); 136 | ASSERT_NO_THROW(table.insert(testColumns1)); 137 | ASSERT_NO_THROW(table.insert(testColumns2)); 138 | 139 | QueryBuilder::Result result; 140 | 141 | // EQ. 142 | char compareStr[100] = "aaabbb"; 143 | 144 | QueryBuilder builder = getBaseBuilder(); 145 | builder.condition(columnMetas[2].name, EQ, compareStr); 146 | ASSERT_NO_THROW(result = builder.execute()); 147 | ASSERT_EQ(result.size(), 1); 148 | compareColumns(testColumns0, result[0].second); 149 | 150 | // GT. 151 | builder = getBaseBuilder(); 152 | builder.condition(columnMetas[2].name, GT, compareStr); 153 | ASSERT_NO_THROW(result = builder.execute()); 154 | ASSERT_EQ(result.size(), 1); 155 | compareColumns(testColumns1, result[0].second); 156 | 157 | // LT. 158 | builder = getBaseBuilder(); 159 | builder.condition(columnMetas[2].name, LT, compareStr); 160 | ASSERT_NO_THROW(result = builder.execute()); 161 | ASSERT_EQ(result.size(), 1); 162 | compareColumns(testColumns2, result[0].second); 163 | } 164 | 165 | TEST_F(QueryConditionTest, TestNullField) { 166 | int value = 1; 167 | 168 | QueryBuilder builder = getBaseBuilder(); 169 | builder.condition(columnMetas[0].name, EQ, (const char *)(&value)); 170 | builder.condition(columnMetas[3].name, EQ, (const char *)(&value)); 171 | 172 | QueryBuilder::Result result; 173 | ASSERT_NO_THROW(result = builder.execute()); 174 | EXPECT_EQ(result.size(), 0); 175 | } 176 | 177 | TEST_F(QueryConditionTest, TestNullOp) { 178 | Columns testColumns0 = {Column(1), Column(1.1F), Column(testVarChar, 100), 179 | Column::nullIntColumn()}; 180 | Columns testColumns1 = {Column(1), Column(1.1F), Column(testVarChar, 100), 181 | Column(1)}; 182 | 183 | ASSERT_NO_THROW(table.insert(testColumns0)); 184 | ASSERT_NO_THROW(table.insert(testColumns1)); 185 | 186 | QueryBuilder::Result result; 187 | 188 | QueryBuilder builder = getBaseBuilder(); 189 | builder.nullCondition(columnMetas[3].name, true); 190 | ASSERT_NO_THROW(result = builder.execute()); 191 | EXPECT_EQ(result.size(), 1); 192 | compareColumns(testColumns0, result[0].second); 193 | 194 | builder = getBaseBuilder(); 195 | builder.nullCondition(columnMetas[3].name, false); 196 | ASSERT_NO_THROW(result = builder.execute()); 197 | EXPECT_EQ(result.size(), 1); 198 | compareColumns(testColumns1, result[0].second); 199 | } 200 | 201 | TEST_F(QueryConditionTest, TestNullValue) { 202 | Columns testColumns0 = {Column(1), Column(1.1F), 203 | Column::nullVarcharColumn(100), Column(2)}; 204 | 205 | ASSERT_NO_THROW(table.insert(testColumns0)); 206 | 207 | std::vector> testCases = { 208 | {EQ, 0}, {NE, 0}, {GT, 0}, {GE, 0}, {LT, 0}, {LE, 0}}; 209 | 210 | QueryBuilder::Result result; 211 | 212 | for (auto &testCase : testCases) { 213 | QueryBuilder builder = getBaseBuilder(); 214 | builder.condition(columnMetas[2].name, testCase.first, ""); 215 | 216 | ASSERT_NO_THROW(result = builder.execute()); 217 | EXPECT_EQ(result.size(), testCase.second); 218 | } 219 | } 220 | 221 | // TEST_F(QueryConditionTest, TestLikeOp) { 222 | // Columns testColumns0 = {Column(1), Column(1.1F), Column("123451", 100), 223 | // Column::nullIntColumn()}; 224 | // Columns testColumns1 = {Column(1), Column(1.1F), Column("012314", 100), 225 | // Column(1)}; 226 | 227 | // ASSERT_NO_THROW(table.insert(testColumns0)); 228 | // ASSERT_NO_THROW(table.insert(testColumns1)); 229 | 230 | // // Matching the first but not second. 231 | // const char *regex = "[1-9]+"; 232 | 233 | // QueryBuilder::Result result; 234 | 235 | // QueryBuilder builder = getBaseBuilder(); 236 | // builder.condition(columnMetas[2].name, LIKE, regex); 237 | 238 | // ASSERT_NO_THROW(result = builder.execute()); 239 | // EXPECT_EQ(result.size(), 1); 240 | // compareColumns(testColumns0, result[0].second); 241 | 242 | // // Test invalid regex. 243 | // regex = "[1-9"; 244 | // builder = getBaseBuilder(); 245 | // builder.condition(columnMetas[2].name, LIKE, regex); 246 | // EXPECT_THROW(result = builder.execute(), Internal::InvalidRegexError); 247 | // } 248 | 249 | // TEST_F(QueryConditionTest, TestInvalidLikeOperator) { 250 | // Columns testColumns0 = {Column(1), Column(1.1F), Column(testVarChar, 251 | // 100), 252 | // Column::nullIntColumn()}; 253 | // ASSERT_NO_THROW(table.insert(testColumns0)); 254 | 255 | // QueryBuilder builder = getBaseBuilder(); 256 | // builder.condition(columnMetas[0].name, LIKE, "[0-9]+"); 257 | 258 | // EXPECT_THROW(auto result = builder.execute(), 259 | // Internal::InvalidOperatorError); 260 | // } 261 | -------------------------------------------------------------------------------- /SimpleDBTest/TableTest.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #ifndef TESTING 3 | #define TESTING 1 4 | #endif 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "Util.h" 11 | 12 | using namespace SimpleDB; 13 | using namespace SimpleDB::Internal; 14 | 15 | class TableTest : public ::testing::Test { 16 | protected: 17 | void SetUp() override { std::filesystem::create_directory("tmp"); } 18 | void TearDown() override { 19 | table.close(); 20 | std::filesystem::remove_all("tmp"); 21 | } 22 | 23 | Table table; 24 | 25 | std::vector columnMetas = { 26 | {.type = INT, .size = 4, .nullable = false, .name = "int_val"}, 27 | {.type = FLOAT, .size = 4, .nullable = false, .name = "float_val"}, 28 | {.type = VARCHAR, 29 | .size = 100, 30 | .nullable = false, 31 | .name = "varchar_val", 32 | .hasDefault = true, 33 | .defaultValue = {.stringValue = "HELLO"}}, 34 | // A nullable column. 35 | {.type = INT, .size = 4, .nullable = true, .name = "int_val_nullable"}, 36 | }; 37 | 38 | const char *testVarChar = "Hello, world"; 39 | 40 | Columns testColumns = {Column(1), Column(1.1F), Column(testVarChar, 100), 41 | Column::nullIntColumn()}; 42 | 43 | const char *tableName = "table_name"; 44 | 45 | void initTable() { 46 | ASSERT_NO_THROW(table.create("tmp/table", tableName, columnMetas)); 47 | } 48 | }; 49 | 50 | TEST_F(TableTest, TestUninitializeAccess) { 51 | EXPECT_THROW(auto _ = table.get({0, 0}), 52 | Internal::TableNotInitializedError); 53 | EXPECT_THROW(table.insert(Columns()), Internal::TableNotInitializedError); 54 | EXPECT_THROW(table.update({0, 0}, Columns()), 55 | Internal::TableNotInitializedError); 56 | EXPECT_THROW(table.remove({0, 0}), Internal::TableNotInitializedError); 57 | } 58 | 59 | TEST_F(TableTest, TestCreateNewTable) { 60 | initTable(); 61 | 62 | EXPECT_EQ(table.meta.numColumn, columnMetas.size()); 63 | EXPECT_EQ(strcmp(tableName, table.meta.name), 0); 64 | } 65 | 66 | TEST_F(TableTest, TestCloseReset) { 67 | initTable(); 68 | table.close(); 69 | 70 | ASSERT_NO_THROW(table.open("tmp/table")); 71 | } 72 | 73 | TEST_F(TableTest, TestInitFromInvalidFile) { 74 | const char *fileName = "tmp/invalid_file"; 75 | PF::create(fileName); 76 | EXPECT_THROW(table.open(fileName), Internal::ReadTableError); 77 | } 78 | 79 | TEST_F(TableTest, TestInitWithDuplicateColumnName) { 80 | std::vector columnMetas = { 81 | {.type = INT, .size = 4, .name = "int_val"}, 82 | {.type = INT, .size = 4, .name = "int_val"}, 83 | }; 84 | EXPECT_THROW(table.create("tmp/table", tableName, columnMetas), 85 | Internal::DuplicateColumnNameError); 86 | } 87 | 88 | TEST_F(TableTest, TestInsertGet) { 89 | initTable(); 90 | 91 | // Write a few pages. 92 | for (int page = 1; page <= 3; page++) { 93 | for (int i = 0; i < table.numSlotPerPage() - 1; i++) { 94 | RecordID id; 95 | ASSERT_NO_THROW(id = table.insert(testColumns)); 96 | 97 | Columns readColumns; 98 | ASSERT_NO_THROW(readColumns = table.get(id)); 99 | compareColumns(testColumns, readColumns); 100 | 101 | // Check if the meta is flushed. 102 | PageHandle handle = PF::getHandle(table.fd, id.page); 103 | EXPECT_TRUE(table.occupied(handle, id.slot)); 104 | } 105 | 106 | EXPECT_EQ(table.meta.firstFree, page + 1); 107 | } 108 | } 109 | 110 | TEST_F(TableTest, TestInsertIncompleteFields) { 111 | initTable(); 112 | // The third column has default value. 113 | Columns newColumns = testColumns; 114 | newColumns.erase(std::next(newColumns.begin(), 2)); 115 | EXPECT_NO_THROW(table.insert(newColumns, /*bitmap=*/0b1011)); 116 | 117 | // Now removes a column without default value. 118 | newColumns.erase(newColumns.begin()); 119 | EXPECT_THROW(table.insert(newColumns, /*bitmap=*/0b1010), 120 | ValueNotGivenError); 121 | } 122 | 123 | TEST_F(TableTest, TestUpdate) { 124 | initTable(); 125 | 126 | // Partial update. 127 | ColumnBitmap bitmap = 0b1101; 128 | Columns newColumns = { 129 | Column(2), Column("Thank you!", 100), 130 | Column(4) // Not null now 131 | }; 132 | 133 | RecordID id; 134 | ASSERT_NO_THROW(id = table.insert(testColumns)); 135 | 136 | ASSERT_NO_THROW(table.update(id, newColumns, bitmap)); 137 | 138 | Columns readColumns; 139 | ASSERT_NO_THROW(table.get(id, readColumns, bitmap)); 140 | compareColumns(newColumns, readColumns); 141 | } 142 | 143 | TEST_F(TableTest, TestRemove) { 144 | initTable(); 145 | 146 | RecordID id; 147 | ASSERT_NO_THROW(id = table.insert(testColumns)); 148 | 149 | PageHandle handle = PF::getHandle(table.fd, id.page); 150 | 151 | ASSERT_NO_THROW(table.remove(id)); 152 | EXPECT_FALSE(table.occupied(handle, id.slot)); 153 | EXPECT_THROW(auto _ = table.get(id), Internal::InvalidSlotError); 154 | } 155 | 156 | TEST_F(TableTest, TestReleasePage) { 157 | initTable(); 158 | 159 | // Write two pages (1, 2). 160 | for (int i = 0; i < 2 * (table.numSlotPerPage() - 1); i++) { 161 | ASSERT_NO_THROW(table.insert(testColumns)); 162 | } 163 | 164 | EXPECT_EQ(table.meta.firstFree, 3); 165 | 166 | // Release a slot from the first page (1). 167 | ASSERT_NO_THROW(table.remove({1, 1})); 168 | 169 | // The first free page should be 1. 170 | ASSERT_EQ(table.meta.firstFree, 1); 171 | } 172 | 173 | TEST_F(TableTest, TestColumnName) { 174 | initTable(); 175 | 176 | std::string readName; 177 | 178 | for (int i = 0; i < columnMetas.size(); i++) { 179 | EXPECT_EQ(table.getColumnIndex(columnMetas[i].name), i); 180 | EXPECT_NO_THROW(readName = table.getColumnName(i)); 181 | EXPECT_EQ(strcmp(readName.c_str(), columnMetas[i].name), 0); 182 | } 183 | 184 | EXPECT_THROW(table.getColumnName(-1), Internal::InvalidColumnIndexError); 185 | EXPECT_THROW(table.getColumnName(columnMetas.size()), 186 | Internal::InvalidColumnIndexError); 187 | } 188 | 189 | TEST_F(TableTest, TestInvalidVarcharSize) { 190 | std::vector columnMetas = { 191 | {.type = VARCHAR, .size = MAX_VARCHAR_LEN + 1, .name = "val"}, 192 | }; 193 | 194 | EXPECT_THROW(table.create("tmp/table", tableName, columnMetas), 195 | Internal::InvalidColumnSizeError); 196 | } 197 | 198 | TEST_F(TableTest, TestMaxVarcharSize) { 199 | char varchar[MAX_VARCHAR_LEN + 1]; 200 | memset(varchar, 'a', MAX_VARCHAR_LEN); 201 | // This is an non-terminated string. 202 | varchar[MAX_VARCHAR_LEN] = 'c'; 203 | 204 | std::vector columnMetas = { 205 | {.type = VARCHAR, .size = MAX_VARCHAR_LEN, .name = "val"}, 206 | }; 207 | 208 | Columns columns = { 209 | Column(varchar, MAX_VARCHAR_LEN), 210 | }; 211 | 212 | RecordID id; 213 | 214 | ASSERT_NO_THROW(table.create("tmp/table", tableName, columnMetas)); 215 | ASSERT_NO_THROW(id = table.insert(columns)); 216 | 217 | Columns readColumns; 218 | ASSERT_NO_THROW(table.get(id, readColumns)); 219 | 220 | // Terminate this to compare. 221 | varchar[MAX_VARCHAR_LEN] = '\0'; 222 | 223 | EXPECT_EQ(strcmp(readColumns[0].data.stringValue, varchar), 0); 224 | } 225 | -------------------------------------------------------------------------------- /SimpleDBTest/Util.cc: -------------------------------------------------------------------------------- 1 | #include "Util.h" 2 | 3 | #include 4 | 5 | void compareColumns(const SimpleDB::Internal::Columns &columns, 6 | const SimpleDB::Internal::Columns &readColumns) { 7 | ASSERT_EQ(readColumns.size(), columns.size()); 8 | for (int i = 0; i < columns.size(); i++) { 9 | EXPECT_EQ(columns[i].type, readColumns[i].type); 10 | EXPECT_EQ(columns[i].size, readColumns[i].size); 11 | if (columns[i].isNull) { 12 | EXPECT_TRUE(readColumns[i].isNull); 13 | continue; 14 | } else { 15 | EXPECT_FALSE(readColumns[i].isNull); 16 | } 17 | if (columns[i].type == SimpleDB::Internal::VARCHAR) { 18 | EXPECT_EQ(memcmp(columns[i].data.stringValue, 19 | readColumns[i].data.stringValue, 20 | strlen(columns[i].data.stringValue)), 21 | 0); 22 | } else { 23 | EXPECT_EQ(memcmp(columns[i].data.stringValue, 24 | readColumns[i].data.stringValue, columns[i].size), 25 | 0); 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /SimpleDBTest/Util.h: -------------------------------------------------------------------------------- 1 | #ifndef _SIMPLEDBTEST_UTIL_H 2 | #define _SIMPLEDBTEST_UTIL_H 3 | 4 | #include 5 | 6 | struct DisableLogGuard { 7 | DisableLogGuard() { 8 | level = SimpleDB::Internal::Logger::getLogLevel(); 9 | SimpleDB::Internal::Logger::setLogLevel(SimpleDB::Internal::SILENT); 10 | } 11 | 12 | ~DisableLogGuard() { SimpleDB::Internal::Logger::setLogLevel(level); } 13 | 14 | private: 15 | SimpleDB::Internal::LogLevel level; 16 | }; 17 | 18 | void compareColumns(const SimpleDB::Internal::Columns &columns, 19 | const SimpleDB::Internal::Columns &readColumns); 20 | 21 | #endif -------------------------------------------------------------------------------- /SimpleDBTest/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int main(int argc, char **argv) { 5 | ::testing::InitGoogleTest(&argc, argv); 6 | SimpleDB::Internal::Logger::setLogLevel(SimpleDB::Internal::SILENT); 7 | return RUN_ALL_TESTS(); 8 | } 9 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 2 | 3 | # googletest 4 | http_archive( 5 | name = "googletest", 6 | urls = ["https://github.com/google/googletest/archive/44c03643cfbc649488a0f437cd18e05f11960d19.zip"], 7 | strip_prefix = "googletest-44c03643cfbc649488a0f437cd18e05f11960d19", 8 | sha256 = "b82dda7e5f9fe2addf972eee106f787b4966d398fcdb06bb0e9942a190e7cfc2" 9 | ) 10 | 11 | # rules_proto_grpc 12 | http_archive( 13 | name = "rules_proto_grpc", 14 | url = "https://github.com/rules-proto-grpc/rules_proto_grpc/archive/4.3.0.tar.gz", 15 | strip_prefix = "rules_proto_grpc-4.3.0", 16 | sha256 = "fb7fc7a3c19a92b2f15ed7c4ffb2983e956625c1436f57a3430b897ba9864059" 17 | ) 18 | load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") 19 | rules_proto_grpc_toolchains() 20 | rules_proto_grpc_repos() 21 | load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") 22 | grpc_deps() 23 | load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") 24 | grpc_extra_deps() 25 | 26 | load("@rules_proto_grpc//cpp:repositories.bzl", rules_proto_grpc_cpp_repos = "cpp_repos") 27 | rules_proto_grpc_cpp_repos() 28 | 29 | load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos") 30 | rules_proto_grpc_python_repos() 31 | 32 | # gflags 33 | http_archive( 34 | name = "com_github_gflags_gflags", 35 | url = "https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.tar.gz", 36 | strip_prefix = "gflags-2.2.2", 37 | sha256 = "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf" 38 | ) 39 | 40 | # rules_python 41 | http_archive( 42 | name = "rules_python", 43 | sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4", 44 | strip_prefix = "rules_python-0.13.0", 45 | url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz", 46 | ) 47 | load("@rules_python//python:pip.bzl", "pip_parse") 48 | 49 | # dep repo 50 | pip_parse( 51 | name = "simpledb_service_dep", 52 | requirements_lock = "//SimpleDBClient:requirements.txt", 53 | ) 54 | load("@simpledb_service_dep//:requirements.bzl", "install_deps") 55 | install_deps() 56 | 57 | # antlr 58 | http_archive( 59 | name = "rules_antlr", 60 | sha256 = "26e6a83c665cf6c1093b628b3a749071322f0f70305d12ede30909695ed85591", 61 | strip_prefix = "rules_antlr-0.5.0", 62 | urls = ["https://github.com/marcohu/rules_antlr/archive/0.5.0.tar.gz"], 63 | ) 64 | load("@rules_antlr//antlr:repositories.bzl", "rules_antlr_dependencies") 65 | load("@rules_antlr//antlr:lang.bzl", "CPP") 66 | rules_antlr_dependencies("4.8", CPP) 67 | 68 | # Hedron's Compile Commands Extractor for Bazel 69 | # https://github.com/hedronvision/bazel-compile-commands-extractor 70 | http_archive( 71 | name = "hedron_compile_commands", 72 | 73 | # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 74 | # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README). 75 | url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/c200ce8b3e0baa04fa5cc3fb222260c9ea06541f.tar.gz", 76 | strip_prefix = "bazel-compile-commands-extractor-c200ce8b3e0baa04fa5cc3fb222260c9ea06541f", 77 | sha256 = "b0a8af42e06108ec62382703daf27f7d8d247fd1b930f249045c70cd9d22f72e" 78 | ) 79 | load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") 80 | hedron_compile_commands_setup() 81 | -------------------------------------------------------------------------------- /docs/client_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liang2kl/simpledb/2d5017d04b1be4136c95436b253de13ec9f93420/docs/client_screenshot.png -------------------------------------------------------------------------------- /docs/dep-graph.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | mygraph 5 | 6 | 7 | 8 | SimpleDBServer 9 | 10 | SimpleDBServer 11 | 12 | 13 | 14 | @gflags 15 | 16 | @gflags 17 | 18 | 19 | 20 | SimpleDBServer->@gflags 21 | 22 | 23 | 24 | 25 | 26 | SimpleDB 27 | 28 | SimpleDB 29 | 30 | 31 | 32 | SimpleDBServer->SimpleDB 33 | 34 | 35 | 36 | 37 | 38 | SQLParser 39 | 40 | SQLParser 41 | 42 | 43 | 44 | SimpleDB->SQLParser 45 | 46 | 47 | 48 | 49 | 50 | SimpleDBService 51 | 52 | SimpleDBService 53 | 54 | 55 | 56 | SimpleDB->SimpleDBService 57 | 58 | 59 | 60 | 61 | 62 | @antlr4-runtime 63 | 64 | @antlr4-runtime 65 | 66 | 67 | 68 | SQLParser->@antlr4-runtime 69 | 70 | 71 | 72 | 73 | 74 | @protobuf 75 | 76 | @protobuf 77 | 78 | 79 | 80 | SimpleDBService->@protobuf 81 | 82 | 83 | 84 | 85 | 86 | @grpc 87 | 88 | @grpc 89 | 90 | 91 | 92 | SimpleDBService->@grpc 93 | 94 | 95 | 96 | 97 | 98 | SimpleDBTest 99 | 100 | SimpleDBTest 101 | 102 | 103 | 104 | SimpleDBTest->SimpleDB 105 | 106 | 107 | 108 | 109 | 110 | @googletest 111 | 112 | @googletest 113 | 114 | 115 | 116 | SimpleDBTest->@googletest 117 | 118 | 119 | 120 | 121 | 122 | SimpleDBClient 123 | 124 | SimpleDBClient 125 | 126 | 127 | 128 | SimpleDBClient->SimpleDBService 129 | 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | # 具体设计 2 | 3 | ## 文件系统/缓存管理 4 | 5 | 支持文件的页式访问及其缓存，使用 LRU 替换策略。提供的主要接口包括： 6 | 7 | - `createFile`：创建文件 8 | - `openFile`：打开文件，返回标识符 9 | - `closeFile`：关闭文件 10 | - `getHandle`：返回对应页的一个“Page Handle”，用于访问页对应的内存及判断缓存是否失效 11 | - `load`：使用 Page Handle 访问页对应的内存，返回对应指针 12 | - `markDirty`：标记脏页 13 | - `renew`：若 Page Handle 因为缓存被逐出而失效，将对应页重新载入缓存 14 | 15 | 使用可以判断失效的 Handle，使得可以在正在使用的页面被替换的情况下保证内存访问的正确性。 16 | 17 | ## 记录管理 18 | 19 | 将表的文件的第一页用于记录表的元数据，第二页及之后的页面用于存储数据。记录采用定长方式，在创建表时根据一行的大小将页面划分为槽，每个槽放置一行数据。 20 | 21 | 在每页起始地址记录页的元数据，包括一个记录槽是否占据的位图，以及下一个空闲的页（链表）。 22 | 23 | 对外提供的主要接口有： 24 | 25 | - `open`：打开文件，加载元数据 26 | - `create`：根据所给 schema 创建空表 27 | - `get`：加载给定 id 的记录并反序列化 28 | - `insert`：序列化给定数据，并寻找空位存储 29 | - `update`：更新给定 id 的记录 30 | - `remove`：删除给定 id 的记录 31 | 32 | ## 索引管理 33 | 34 | 使用 B+ 树进行记录的索引。由于需要支持 NULL key 和重复 key 的索引，将 `(key, isNull, recordId)` 三元组作为索引的键进行索引。 35 | 36 | 具体的存储上，每页对应 B+ 树的一个节点。由于 B+ 树存在叶子节点和内部节点的区别，为了和二进制存储对应，将两种节点的共同数据放置于开头，节点独有的数据放置在之后，即： 37 | 38 | ```c++ 39 | struct SharedNode { 40 | bool isLeaf; 41 | ... 42 | }; 43 | struct InnerNode { 44 | SharedNode shared; 45 | ... 46 | }; 47 | struct LeafNode { 48 | SharedNode shared; 49 | ... 50 | } 51 | ``` 52 | 53 | 这使得我们从内存中读入时可以根据 `SharedNode` 中的 `isLeaf` 判断节点类型，并转换对应的类型： 54 | 55 | ```c++ 56 | char *data = ...; 57 | SharedNode *node = (SharedNode *)data; 58 | if (node->isLeaf) { 59 | LeafNode *leaf = (LeafNode *)node; 60 | } else { 61 | InnerNode *inner = (InnerNode *)node; 62 | } 63 | ``` 64 | 65 | 索引的插入根据标准的 B+ 树实现，删除实现为 lazy remove，只作 invalid 的标记。 66 | 67 | 另外，叶子结点存储指向下一叶子结点的指针，方便 range query。 68 | 69 | 提供的主要接口有： 70 | 71 | - `open`：打开索引 72 | - `create`：创建索引 73 | - `insert`：插入索引项 74 | - `remove`：删除索引项 75 | - `iterateRange`：遍历特定范围内的索引项 76 | 77 | ## 系统管理 78 | 79 | 系统管理主要涉及几张系统表： 80 | 81 | - `databases`：记录所有创建的数据库 82 | - `tables`：记录所有创建的表（schema 直接存在表文件头中） 83 | - `indexes`：记录所有创建的索引 84 | - `foreign_key`：记录所有创建的外键 85 | 86 | 由于进行查询时这几张表频繁使用，因此常驻内存中。 87 | 88 | ## 查询处理 89 | 90 | 因需要支持种类繁多的 `SELECT` 语句，并且可能需要利用索引进行查询，因此需要设计一套通用性较强的查询抽象。 91 | 92 | 主要的抽象有 `QueryDataSource`、`QueryFilter` 和 `QueryBuilder`。Data source 提供查询数据，Filter 对数据进行筛选和转换，而 Builder 是提供给外界的查询接口。 93 | 94 | `QueryDataSource` 为抽象类，需要实现的主要接口包括： 95 | 96 | - `iterate`：遍历此数据源所有的记录，可随时停止 97 | - `getColumnInfo`：返回类似于 schema 的信息（因涉及到 JOIN 和原本打算实现的嵌套查询，不能简单地使用表本身的 schema） 98 | 99 | `QueryFilter` 负责对遍历的记录进行筛选，返回 (是否继续遍历，是否接受此记录)。Filter 包括： 100 | 101 | - `ValueConditionFilter`：处理与字面值进行比较的条件 102 | - `NullConditionFilter`：处理是否为 NULL 的条件 103 | - `ColumnConditionFilter`：处理两个列之间的比较 104 | - `SelectFilter`：处理选择部分列的语句以及聚合查询 105 | - `LimitFilter`：处理 `LIMIT` 106 | - `OffsetFilter`：处理 `OFFSET` 107 | 108 | `QueryBuilder` 根据所有的条件，转换为对应的 Filter，并将其相连接。当执行查询时，数据从数据源出发，依次通过各个 Filter，当所有的 Filter 通过后，将其加入到结果中，否则丢弃。这样做避免了重复对整个数据集进行筛选，大大减少了内存使用。 109 | 110 | 在这套抽象的基础上，很容易实现 JOIN 和索引加速的查询，只需要实现对应的 Data source，给出遍历的方法即可（对应代码中的 `JoinedTable` 和 `IndexedTable`），而 Filter 是通用的。`QueryBuilder` 因为只需要用到 `QueryDataSource` 抽象类的接口，因此可以接受任意的 Data source，无论是原始的 `Table`，使用索引的 `IndexedTable`，还是多表连接的 `JoinedTable`。 111 | 112 | 另外，`QueryBuilder` 本身也可作为 Data source，可用来遍历符合条件的记录，从而可以直接用来实现 `DELETE` 和 `UPDATE` 的条件判断，以及支持嵌套查询（虽然未实现）。 113 | 114 | ## 错误处理及 Logger 115 | 116 | 错误处理不使用返回值，而是使用 `throw` 实现。这样最明显的好处在于，无论在多深的调用栈发生错误，都可以在任意想要处理错误的位置捕捉到，并根据对应的错误类型进行处理，而不需要在任何可能发生错误的调用处都进行处理。 117 | 118 | 错误类型包括内部使用的错误和对外暴露的错误，内部使用的错误在进行 SQL 执行时会被转换为暴露的错误并返回，因此客户端可以支持具体错误类型的显示。 119 | 120 | 另外，实现了一套 logging 系统，根据设定的重要程度打印日志。在各层的关键调用处使用 `VERBOSE` 等级打印调用信息，对调试有很大的帮助。 121 | -------------------------------------------------------------------------------- /docs/server_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liang2kl/simpledb/2d5017d04b1be4136c95436b253de13ec9f93420/docs/server_screenshot.png --------------------------------------------------------------------------------