├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── cli ├── .gitignore ├── index.ts ├── package.json ├── tsconfig.json └── yarn.lock ├── client ├── .gitignore ├── .npmignore ├── README.md ├── index.ts ├── package.json ├── src │ ├── client.ts │ ├── config.ts │ └── types.ts ├── test.ts ├── tsconfig.json └── yarn.lock ├── db ├── .gitignore ├── BufferManager │ ├── BufferManager.cpp │ ├── BufferManager.h │ └── Readme.md ├── CMakeLists.txt ├── CatalogManager │ ├── CatalogManager.cpp │ ├── CatalogManager.h │ └── README.md ├── IndexManager │ ├── BPInclude.h │ ├── BPTree.cpp │ ├── BPTree.h │ ├── BPTreeNode.cpp │ ├── BPTreeNode.h │ ├── IndexManager.cpp │ ├── IndexManager.h │ └── Readme.md ├── README.md ├── RecordManager │ ├── Readme.md │ ├── RecordManager.cpp │ └── RecordManager.h ├── api │ ├── README.md │ ├── api.cpp │ ├── api.h │ ├── utils.cpp │ └── utils.h ├── external_main │ ├── external_main.cpp │ └── external_main.h ├── go-svc │ ├── README.md │ ├── main.go │ └── placeholder.cpp ├── go.mod ├── go.sum ├── interpreter │ ├── README.md │ ├── init.sh │ ├── interpreter.cpp │ ├── interpreter.h │ ├── parser.y │ ├── readline.c │ └── tokenizer.l ├── macro.h ├── main.cpp └── utils │ ├── exception.cpp │ ├── exception.h │ ├── utils.cpp │ └── utils.h └── master ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── README.md ├── mvnw ├── mvnw.cmd ├── pom.xml └── src ├── main └── java │ └── xyz │ └── ralxyz │ └── minisql_master │ ├── Cluster.java │ ├── Config.java │ ├── Controller.java │ ├── MiniSqlMasterApplication.java │ └── model │ ├── RegionInfo.java │ ├── SqlResponse.java │ └── Statement.java └── test └── java └── xyz └── ralxyz └── minisql_master └── MiniSqlMasterApplicationTests.java /.gitattributes: -------------------------------------------------------------------------------- 1 | *.h linguist-language=C++ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/region.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ![MiniSQL](https://raw.githubusercontent.com/RalXYZ/repo-pictures/main/MiniSQL/minisql_overall.png) 4 | 5 | # MiniSQL 6 | 7 | **Disclaimer: This project is the final project of a course, which can only be considered as a toy. It shows some undergraduates' ideas on how to build a proper database. Suggestions are welcomed.** 8 | 9 | MiniSQL is designed to be a distributed relational database system. SQL parsing, execution and storing are implemented in `./db`, we called it a *MiniSQL Region*. Since region is written in compiled languages, it can be compiled into an executable, and run independently. Region can also be linked with a "web service shell" written in golang, which forms a *Region Server*, that is, a region with HTTP service. Master, implemented in `./master`, has the ability to manage multiple region servers, because the state of all nodes are stored in Apache Zookeeper. Sadly, since this project is only a demo written by some undergraduates, It only supports one master, $m$ regions and $n$ replication of data, where $m$ is a arbitrary positive integer, and $n < m$. The system introduced above is called *MiniSQL Cluster*. 10 | To Interact with the cluster, we implemented a NPM client in `./client`. Since client is just functions that sends and receive messages, to make the whole project usable, we implemented a cli in `./cli`, which relies on the npm client to interact with the cluster. 11 | 12 | The picture above shows the architecture of MiniSQL cluster. To be honest, we didn't implement everything shown on this picture, but surely, you can add some features based on our current implementation to achieve these features without changing the architecture. Some unimplemented features can be implemented only using less than a fifty lines of code. We didn't implement it because as undergraduates, we do not have enough time to implement everything. 13 | - Figure one shows that many clients can connect to MiniSQL cluster. Although we only implemented a npm client, but implementing clients using other languages is just a matter of time. 14 | - Figure two shows the architecture inside the cluster, which has been talked about in `./master`. 15 | - Figure three shows the relationship between master and region. The relationship between the same types of nodes are equivalent. Although we only support one master node currently, with the addition of some locks and interacting with Zookeeper, we surly can support multiple masters nodes. 16 | - Figure four shows that different nodes can exist on different machines. Although in our current implementation, nodes can only exist on one machine, the only difference between starting a process locally or remotely is whether we need to build a tunnel, for example using *ssh*. 17 | 18 | In each directory, there exists detailed descriptions about the corresponding module. -------------------------------------------------------------------------------- /cli/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.js 3 | yarn-error.log -------------------------------------------------------------------------------- /cli/index.ts: -------------------------------------------------------------------------------- 1 | import MiniSQLClient from '@enzymii/minisql-client'; 2 | import Inquirer from 'inquirer'; 3 | import Chalk from 'chalk'; 4 | 5 | type Answer = { 6 | sql: string; 7 | table: string; 8 | isSelect: boolean; 9 | }; 10 | 11 | (async () => { 12 | const client = await MiniSQLClient.create(); 13 | while (true) { 14 | try { 15 | const { sql, table, isSelect } = await Inquirer.prompt([ 16 | { 17 | type: 'input', 18 | name: 'sql', 19 | message: 'Please input your SQL:', 20 | }, 21 | { 22 | type: 'input', 23 | name: 'table', 24 | message: 'Please input the table name:', 25 | }, 26 | { 27 | type: 'confirm', 28 | name: 'isSelect', 29 | message: 'Is this a select statement?', 30 | default: false, 31 | }, 32 | ]); 33 | 34 | const { code, msg } = isSelect 35 | ? await client.queryRegion(table, sql) 36 | : await client.queryMaster(table, sql); 37 | 38 | if (code === 0) { 39 | console.log(msg); 40 | } else if (code > 0) { 41 | console.log(Chalk.cyanBright(msg)); 42 | } else { 43 | console.log(Chalk.redBright(msg)); 44 | } 45 | 46 | if (sql.toLocaleLowerCase() === 'exit;') { 47 | break; 48 | } 49 | } catch (e) {} 50 | } 51 | })(); 52 | -------------------------------------------------------------------------------- /cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "minisql-cli", 3 | "version": "1.0.0", 4 | "description": "cli for minisql", 5 | "main": "index.js", 6 | "author": "enzymii ", 7 | "license": "MIT", 8 | "type": "module", 9 | "scripts": { 10 | "start": "tsc && node --experimental-specifier-resolution=node index" 11 | }, 12 | "dependencies": { 13 | "@enzymii/minisql-client": "^1.0.1", 14 | "chalk": "^5.0.1", 15 | "inquirer": "^8.2.4" 16 | }, 17 | "devDependencies": { 18 | "@types/inquirer": "^8.2.1", 19 | "typescript": "^4.6.4" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /cli/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Basic Options */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, 8 | "module": "ESNext" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, 9 | // "lib": [], /* Specify library files to be included in the compilation. */ 10 | // "allowJs": true, /* Allow javascript files to be compiled. */ 11 | // "checkJs": true, /* Report errors in .js files. */ 12 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 15 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 16 | // "outFile": "./", /* Concatenate and emit output to single file. */ 17 | // "outDir": "./", /* Redirect output structure to the directory. */ 18 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 19 | // "composite": true, /* Enable project compilation */ 20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 21 | // "removeComments": true, /* Do not emit comments to output. */ 22 | // "noEmit": true, /* Do not emit outputs. */ 23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 26 | 27 | /* Strict Type-Checking Options */ 28 | "strict": true /* Enable all strict type-checking options. */, 29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 30 | // "strictNullChecks": true, /* Enable strict null checks. */ 31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 36 | 37 | /* Additional Checks */ 38 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 40 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 41 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 42 | 43 | /* Module Resolution Options */ 44 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 45 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 46 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 47 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 48 | // "typeRoots": [], /* List of folders to include type definitions from. */ 49 | // "types": [], /* Type declaration files to be included in compilation. */ 50 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 51 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 52 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 53 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 54 | 55 | /* Source Map Options */ 56 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 57 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 58 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 59 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 60 | 61 | /* Experimental Options */ 62 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 63 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 64 | 65 | /* Advanced Options */ 66 | "skipLibCheck": true /* Skip type checking of declaration files. */, 67 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /client/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.js 3 | *.d.ts 4 | dist/ -------------------------------------------------------------------------------- /client/.npmignore: -------------------------------------------------------------------------------- 1 | test* -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | # MiniSQL-Client 2 | 3 | Cache region's address, send read-only request to region server and write statements to master server. 4 | 5 | ## Warning 6 | 7 | _It is used for course assignment only, you may not use it in production environment~_ 8 | 9 | As there is file operation, it may not use in a browser environment. 10 | 11 | When you run it in Node.js, please make sure to set `--experimental-specifier-resolution=node` flag to ensure proper module resolution. 12 | 13 | ## Usage 14 | 15 | _Note that all functions implemented in the package are asynchronous~_ 16 | 17 | To create an instance of the client: 18 | 19 | ```ts 20 | const client = await MiniSQLClient.create('xxx'); 21 | ``` 22 | 23 | Where `xxx` is the master server's address. 24 | 25 | To send commands, use 26 | 27 | ```ts 28 | await client.queryRegion([tableName], [sql]); 29 | ``` 30 | 31 | or 32 | 33 | ```ts 34 | await client.queryMaster([tableName], [sql]); 35 | ``` 36 | 37 | Note that the first one is for read-only queries, the second one is for write queries. 38 | -------------------------------------------------------------------------------- /client/index.ts: -------------------------------------------------------------------------------- 1 | import MiniSQLClient from "./src/client"; 2 | 3 | export default MiniSQLClient; -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@enzymii/minisql-client", 3 | "version": "1.1.0", 4 | "description": "HTTP client for distributed MiniSQL", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "author": "enzymii ", 8 | "license": "MIT", 9 | "type": "module", 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/BMS-2021/MiniSQL" 13 | }, 14 | "scripts": { 15 | "prepare": "tsc", 16 | "test": "tsc && node --experimental-specifier-resolution=node dist/test" 17 | }, 18 | "devDependencies": { 19 | "@types/node": "^17.0.31", 20 | "typescript": "^4.6.4" 21 | }, 22 | "dependencies": { 23 | "@types/inquirer": "^8.2.1", 24 | "axios": "^0.27.2", 25 | "chalk": "^5.0.1", 26 | "inquirer": "^8.2.4" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /client/src/client.ts: -------------------------------------------------------------------------------- 1 | import fs, { mkdirSync } from 'fs'; 2 | import Axios from 'axios'; 3 | import chalk from 'chalk'; 4 | import inquirer from 'inquirer'; 5 | import type { AxiosInstance } from 'axios'; 6 | 7 | // may not remove '.js' because of strange module resolution strategy 8 | import config from './config'; 9 | 10 | import { RegionInfo, Request, SqlResponse } from './types'; 11 | 12 | class MiniSQLClient { 13 | private cache?: RegionInfo[]; 14 | private masterUrl!: string; 15 | private http!: AxiosInstance; 16 | 17 | public static async create(masterUrl?: string) { 18 | if (!masterUrl) { 19 | const { masterUrl: inputUrl } = await inquirer.prompt<{ 20 | masterUrl: string; 21 | }>([ 22 | { 23 | type: 'input', 24 | name: 'masterUrl', 25 | message: 'Please input master url:', 26 | default: '[http://]ip:port', 27 | validate: (input) => 28 | /^(https?:\/\/)?[a-z0-9.]+(:\d+)?$/.test(input) || 29 | /:?\d+/.test(input) || 30 | chalk.red('Invalid URL'), 31 | }, 32 | ]); 33 | masterUrl = inputUrl; 34 | } 35 | 36 | if (!/^http/.test(masterUrl)) { 37 | masterUrl = 'http://' + masterUrl; 38 | } 39 | 40 | const client = new MiniSQLClient(masterUrl); 41 | await client.loadCache(); 42 | return client; 43 | } 44 | 45 | private constructor(masterUrl: string) { 46 | this.masterUrl = masterUrl; 47 | this.http = Axios.create({ proxy: false }); 48 | } 49 | 50 | private async loadCache() { 51 | try { 52 | console.log(chalk.cyan('Loading Cache...')); 53 | if (fs.existsSync(config.regionCache)) { 54 | const cache = fs.readFileSync(config.regionCache); 55 | this.cache = JSON.parse(cache.toString('utf-8')); 56 | } else { 57 | await this.flushCache(); 58 | } 59 | console.log(chalk.green('Load cache ok!')); 60 | } catch (e) { 61 | this.exceptionHandler(e); 62 | } 63 | } 64 | 65 | private async flushCache() { 66 | try { 67 | console.log(chalk.blueBright('Flushing Cache...')); 68 | const newCache = await this.request({ 69 | url: this.masterUrl + '/cache', 70 | }); 71 | fs.writeFileSync( 72 | config.regionCache, 73 | Buffer.from(JSON.stringify(newCache)) 74 | ); 75 | this.cache = newCache; 76 | console.log( 77 | chalk.green('Flush cache ok with: '), 78 | JSON.stringify(newCache) 79 | ); 80 | } catch (e) { 81 | this.exceptionHandler(e); 82 | } 83 | } 84 | 85 | async queryRegion(tableName: string, sql: string): Promise { 86 | console.log(chalk.magenta('Querying To Region Server...')); 87 | return this._queryRegion(tableName, sql, 3); 88 | } 89 | 90 | private async _queryRegion( 91 | tableName: string, 92 | sql: string, 93 | flushAndRetryTimes = 3 94 | ): Promise { 95 | if (flushAndRetryTimes < 0) { 96 | throw 'Max Retry Times Exceeded'; 97 | } 98 | try { 99 | let resp = undefined; 100 | if (this.cache) { 101 | const targetRegions = this.cache.filter(({ tables }) => 102 | tables.find( 103 | (table) => 104 | table.toLocaleLowerCase() === tableName.toLocaleLowerCase() 105 | ) 106 | ); 107 | if (targetRegions.length > 0) { 108 | // load balancing 109 | const index = Math.floor(Math.random() * targetRegions.length); 110 | const targetRegion = targetRegions[index]; 111 | 112 | console.log( 113 | chalk.yellow(`Select Region ${targetRegion.regionUrl} as target~`) 114 | ); 115 | 116 | resp = await this.request({ 117 | url: 'http://' + targetRegion.regionUrl, 118 | method: 'POST', 119 | data: { command: sql }, 120 | }); 121 | } 122 | } 123 | if (!resp || resp.code != 0) { 124 | await this.flushCache(); 125 | resp = await this._queryRegion(tableName, sql, flushAndRetryTimes - 1); 126 | } 127 | return resp; 128 | } catch (e) { 129 | this.exceptionHandler(e); 130 | } 131 | } 132 | 133 | async queryMaster(tableName: string, sql: string): Promise { 134 | try { 135 | console.log(chalk.magenta('Sending request to master...')); 136 | return await this.request({ 137 | url: this.masterUrl + '/statement', 138 | method: 'POST', 139 | data: { tableName, command: sql }, 140 | }); 141 | } catch (e) { 142 | this.exceptionHandler(e); 143 | } 144 | } 145 | 146 | private async request({ 147 | url = this.masterUrl, 148 | method = 'GET', 149 | params, 150 | data, 151 | contentType = 'application/json', 152 | }: Request) { 153 | try { 154 | const res = await this.http.request({ 155 | url, 156 | method, 157 | params, 158 | data, 159 | headers: { 'Content-Type': contentType }, 160 | }); 161 | 162 | if (res.status.toString().startsWith('2')) { 163 | const { data } = res; 164 | return data; 165 | } 166 | throw res; 167 | } catch (e) { 168 | this.exceptionHandler(e); 169 | } 170 | } 171 | 172 | private exceptionHandler(e: any): never { 173 | if (e.response) { 174 | console.log(chalk.redBright.bold('AXIOS RESPONSE ERROR')); 175 | console.log(chalk.red('data: '), e.response.data); 176 | console.log(chalk.red('status: '), e.response.data); 177 | } else if (e.request) { 178 | console.log(chalk.redBright.bold('AXIOS REQUEST ERROR')); 179 | console.log(chalk.red('request: '), e.request); 180 | } else if (e.message) { 181 | console.log(chalk.redBright.bold('AXIOS UNKNOWN ERROR')); 182 | console.log(chalk.red('err msg: '), e.message); 183 | } else { 184 | console.log(chalk.redBright.bold('OTHER ERROR')); 185 | console.log(chalk.red(e)); 186 | } 187 | // process.exit(-1); 188 | throw 'MiniSQL Client Exception'; 189 | } 190 | } 191 | 192 | export default MiniSQLClient; 193 | -------------------------------------------------------------------------------- /client/src/config.ts: -------------------------------------------------------------------------------- 1 | export default { 2 | regionCache: './region.json', 3 | }; 4 | -------------------------------------------------------------------------------- /client/src/types.ts: -------------------------------------------------------------------------------- 1 | type RegionInfo = { 2 | tables: string[]; 3 | regionUrl: string; 4 | }; 5 | 6 | type Request = { 7 | url?: string; 8 | method?: 'GET' | 'POST' | 'PUT' | 'DELETE'; 9 | params?: any; 10 | data?: any; 11 | contentType?: 'application/json' | 'text/plain'; 12 | }; 13 | 14 | type SqlResponse = { 15 | code: number; 16 | msg: string; 17 | }; 18 | 19 | export { RegionInfo, Request, SqlResponse }; 20 | -------------------------------------------------------------------------------- /client/test.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import MiniSQLClient from './src/client'; 3 | 4 | const main = async () => { 5 | const myClient = await MiniSQLClient.create('http://localhost:8080'); 6 | const req = await myClient.queryRegion('foo', 'SELECT * FROM foo;'); 7 | console.log(req.msg); 8 | }; 9 | 10 | main(); 11 | -------------------------------------------------------------------------------- /client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "experimentalDecorators": true, 4 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 5 | 6 | /* Projects */ 7 | // "incremental": true, /* Enable incremental compilation */ 8 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 9 | // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ 10 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ 11 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 12 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 13 | 14 | /* Language and Environment */ 15 | "target": "ESNext" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, 16 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ 17 | // "jsx": "preserve", /* Specify what JSX code is generated. */ 18 | // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ 19 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 20 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ 21 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 22 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ 23 | // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ 24 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 25 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 26 | 27 | /* Modules */ 28 | "module": "ESNext" /* Specify what module code is generated. */, 29 | // "rootDir": "./", /* Specify the root folder within your source files. */ 30 | "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, 31 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 32 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 33 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 34 | // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ 35 | // "types": [], /* Specify type package names to be included without being referenced in a source file. */ 36 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 37 | "resolveJsonModule": true /* Enable importing .json files */, 38 | // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ 39 | 40 | /* JavaScript Support */ 41 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ 42 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ 43 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ 44 | 45 | /* Emit */ 46 | "declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */, 47 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 48 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 49 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 50 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ 51 | "outDir": "./dist" /* Specify an output folder for all emitted files. */, 52 | // "removeComments": true, /* Disable emitting comments. */ 53 | // "noEmit": true, /* Disable emitting files from a compilation. */ 54 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 55 | // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ 56 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 57 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 59 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 60 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 61 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 62 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 63 | // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ 64 | // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ 65 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 66 | // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ 67 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 68 | // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ 69 | 70 | /* Interop Constraints */ 71 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ 72 | // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ 73 | "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, 74 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 75 | "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, 76 | 77 | /* Type Checking */ 78 | "strict": true /* Enable all strict type-checking options. */, 79 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ 80 | // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ 81 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 82 | // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ 83 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 84 | // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ 85 | // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ 86 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 87 | // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ 88 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ 89 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 90 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 91 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 92 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 93 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 94 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ 95 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 96 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 97 | 98 | /* Completeness */ 99 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 100 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /db/.gitignore: -------------------------------------------------------------------------------- 1 | cmake-build-debug 2 | .idea 3 | .DS_Store 4 | .vs 5 | .vscode 6 | out/ 7 | IndexManager/main.cpp 8 | cmake-build-debug-wsl -------------------------------------------------------------------------------- /db/BufferManager/BufferManager.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Apple on 2021/6/12. 3 | // 4 | 5 | #include "BufferManager.h" 6 | #include "../macro.h" 7 | #include 8 | #include 9 | #include 10 | 11 | #include "../utils/exception.h" 12 | 13 | using namespace std; 14 | 15 | File::File(const string& _filename) { 16 | type = 0; 17 | filename = _filename; 18 | blockCnt = 0; 19 | 20 | struct stat st; 21 | if (stat(filename.c_str(), &st) == 0) { 22 | totBlockCnt = (st.st_size / macro::BlockSize); 23 | } else { 24 | totBlockCnt = 0; 25 | } 26 | if(totBlockCnt <= 0) totBlockCnt = 1; 27 | 28 | next = nullptr; 29 | firstBlock = nullptr; 30 | } 31 | 32 | 33 | BufferManager::BufferManager() { 34 | LRUNum = 0; 35 | fileCnt = 0; 36 | fileHandle = nullptr; 37 | blockHandle = nullptr; 38 | } 39 | 40 | Block &BufferManager::writeBlock(Block &block) { 41 | fstream fp; 42 | fp.open(block.file->filename, ios::in | ios::out | ios::ate | ios::binary); 43 | fp.seekg(block.blockID * macro::BlockSize, ios::beg); 44 | fp.write(block.blockContent, macro::BlockSize); 45 | fp.close(); 46 | memset(block.blockContent, 0, macro::BlockSize); 47 | return block; 48 | } 49 | 50 | Block &BufferManager::resetBlock(Block &block) { 51 | block.next = nullptr; 52 | block.dirty = 0; 53 | memset(block.blockContent, 0, macro::BlockSize); 54 | return block; 55 | } 56 | 57 | void BufferManager::replace(File &file, Block &block) { 58 | block.file = &file; 59 | Block* curBlock = file.firstBlock; 60 | if(!curBlock) file.firstBlock = █ 61 | else { 62 | block.next = file.firstBlock; 63 | file.firstBlock = █ 64 | } 65 | } 66 | 67 | Block &BufferManager::getFreeBlock(File &file) { 68 | if(!blockHandle) { 69 | if(file.blockCnt < macro::MaxBlocks) { 70 | file.blockCnt++; 71 | Block* blockPtr = new Block; 72 | resetBlock(*blockPtr); 73 | replace(file, *blockPtr); 74 | return *blockPtr; 75 | } 76 | Block &b = getLRUBlock(file); 77 | if(b.dirty) { 78 | writeBlock(b); 79 | } 80 | return b; 81 | } 82 | 83 | Block &b = *blockHandle; 84 | blockHandle = blockHandle->next; 85 | return b; 86 | } 87 | 88 | Block &BufferManager::getLRUBlock(File &file){ 89 | int minLRU = LRUNum; 90 | Block *detect = nullptr; 91 | Block* curBlock = file.firstBlock; 92 | while(curBlock) { 93 | if (curBlock->LRUCount < minLRU) { 94 | minLRU = curBlock->LRUCount; 95 | detect = curBlock; 96 | } 97 | curBlock = curBlock->next; 98 | } 99 | return *detect; 100 | } 101 | 102 | void BufferManager::closeFile(File *file) { 103 | if(!file) return; 104 | Block *curBlock = file->firstBlock; 105 | Block *nextBlock = nullptr; 106 | while(curBlock) { 107 | nextBlock = curBlock->next; 108 | if(curBlock->dirty) writeBlock(*curBlock); 109 | curBlock = nextBlock; 110 | } 111 | 112 | Block *bh = blockHandle; 113 | if(!bh) blockHandle = file->firstBlock; 114 | else { 115 | while(bh->next) bh = bh->next; 116 | bh->next = file->firstBlock; 117 | } 118 | fileCnt--; 119 | } 120 | 121 | void BufferManager::closeAllFile() { 122 | File *curFile = fileHandle; 123 | while(curFile) { 124 | closeFile(curFile); 125 | curFile = curFile->next; 126 | } 127 | } 128 | 129 | File &BufferManager::getFile(const string& filename) { 130 | File *preFile = nullptr; 131 | File *curFile = fileHandle; 132 | while(curFile && curFile->filename != filename) { 133 | preFile = curFile; 134 | curFile = curFile->next; 135 | } 136 | if(!curFile) { //file not in memory 137 | if(fileCnt == macro::MaxFiles) closeFile(preFile); 138 | File* newFile = new File(filename); 139 | newFile->filename = filename; 140 | newFile->next = fileHandle; 141 | fileCnt++; 142 | fileHandle = newFile; 143 | curFile = newFile; 144 | } 145 | return *curFile; 146 | } 147 | 148 | Block &BufferManager::readBlock(const string& filename, int blockID) { 149 | fstream fp; 150 | fp.open(filename, ios::in | ios::out | ios::binary); 151 | if (!fp.good()) { 152 | throw sql_exception(401, "buffer manager", "element not found"); 153 | } 154 | 155 | File &file = getFile(filename); 156 | Block &block = getFreeBlock(file); 157 | block.blockID = blockID; 158 | 159 | if(file.totBlockCnt <= blockID) { 160 | file.totBlockCnt = blockID + 1; 161 | memset(block.blockContent, 0, macro::BlockSize); 162 | } else { 163 | fp.seekg(blockID * macro::BlockSize, ios::beg); 164 | fp.read(block.blockContent, macro::BlockSize); 165 | } 166 | 167 | return block; 168 | } 169 | 170 | Block &BufferManager::getBlock(const string& filename, int blockID) { 171 | File &file = getFile(filename); 172 | Block *curBlock = file.firstBlock; 173 | while(curBlock && curBlock->blockID != blockID) { 174 | curBlock = curBlock->next; 175 | } 176 | if(!curBlock) { //block not found 177 | curBlock = &readBlock(filename, blockID); 178 | } 179 | lock(*curBlock); 180 | return *curBlock; 181 | } 182 | 183 | Block &BufferManager::findBlock(const string& filename, int blockID) const { 184 | File *curFile = fileHandle; 185 | while(curFile && curFile->filename != filename) { 186 | curFile = curFile->next; 187 | } 188 | if(!curFile) { 189 | throw sql_exception(401, "buffer manager", "fail to open file: " + filename); 190 | } 191 | 192 | Block *curBlock = curFile->firstBlock; 193 | while(curBlock && curBlock->blockID != blockID) { 194 | curBlock = curBlock->next; 195 | } 196 | if(!curBlock) { 197 | throw sql_exception(401, "buffer manager", "element not found"); 198 | } 199 | 200 | return *curBlock; 201 | } 202 | 203 | void BufferManager::setDirty(const string &filename, unsigned int blockID) { 204 | Block &block = findBlock(filename, blockID); 205 | block.dirty = true; 206 | } 207 | 208 | void BufferManager::unlock(const string& filename, unsigned int blockID) { 209 | Block &block = findBlock(filename, blockID); 210 | block.lock = false; 211 | } 212 | 213 | void BufferManager::lock(Block &block) { 214 | LRUNum++; 215 | block.lock = true; 216 | block.LRUCount = LRUNum; 217 | } 218 | 219 | void BufferManager::createFile(const string& in) { 220 | ofstream f1(in); 221 | } 222 | 223 | void BufferManager::removeFile(const string& filename) { 224 | File *curFile = fileHandle; 225 | while(curFile && curFile->filename != filename) { 226 | curFile = curFile->next; 227 | } 228 | if(curFile) closeFile(curFile); 229 | if (remove(filename.c_str())) { 230 | throw sql_exception(402, "buffer manager", "fail to remove file: " + filename); 231 | } 232 | } 233 | 234 | int BufferManager::getBlockCnt(const string& filename) { 235 | return getFile(filename).totBlockCnt; 236 | return 0; 237 | } -------------------------------------------------------------------------------- /db/BufferManager/BufferManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Apple on 2021/6/12. 3 | // 4 | 5 | #ifndef MINISQL_BUFFERMANAGER_H 6 | #define MINISQL_BUFFERMANAGER_H 7 | 8 | #include 9 | #include 10 | #include "../macro.h" 11 | 12 | using namespace std; 13 | 14 | typedef struct File File; 15 | typedef struct Block Block; 16 | typedef struct BufferManager BufferManager; 17 | 18 | struct File { 19 | int type; //0:db file 1:table file 2: index file 20 | 21 | string filename; //the name of the file 22 | int blockCnt; 23 | int totBlockCnt; 24 | File *next; //the pointer points to the next file 25 | Block *firstBlock; 26 | 27 | File(const string& _filename); 28 | 29 | }; 30 | 31 | struct Block { 32 | int blockID; 33 | bool dirty; //whether the block been changed 34 | Block *next; 35 | File *file; 36 | int usedSize; 37 | int LRUCount; 38 | int lock; //prevent the block from replacing 39 | char *blockContent; 40 | 41 | Block() { 42 | blockID = 0; 43 | dirty = 0; 44 | next = nullptr; 45 | file = nullptr; 46 | usedSize = 0; 47 | LRUCount = 0; 48 | lock = 0; 49 | blockContent = static_cast(calloc(1, macro::BlockSize)); 50 | 51 | } 52 | 53 | Block(bool isNull) { 54 | blockID = 0; 55 | dirty = 0; 56 | next = nullptr; 57 | file = nullptr; 58 | usedSize = 0; 59 | LRUCount = 0; 60 | lock = 0; 61 | if(isNull) blockContent = nullptr; 62 | } 63 | ~Block() { 64 | // if(blockContent) free(blockContent); 65 | } 66 | }; 67 | 68 | class BufferManager { 69 | public: 70 | BufferManager(); 71 | 72 | ~BufferManager() = default; 73 | 74 | void closeAllFile(); 75 | 76 | Block &getBlock(const string& fileName, int blockID); 77 | 78 | void setDirty(const string &filename, unsigned int blockID); 79 | 80 | void unlock(const string& filename, unsigned int blockID); 81 | 82 | static void createFile(const string& in); 83 | 84 | void removeFile(const string& filename); 85 | 86 | int getBlockCnt(const string& filename); 87 | 88 | Block nullBlock = Block(true); 89 | File *fileHandle; 90 | private: 91 | void closeFile(File *file); 92 | 93 | static Block &writeBlock(Block &block); 94 | 95 | static Block &resetBlock(Block &block); 96 | 97 | static void replace(File &file, Block &block); 98 | 99 | void lock(Block &block); 100 | 101 | Block &getFreeBlock(File &file); 102 | 103 | Block &getLRUBlock(File &file); 104 | 105 | Block &readBlock(const string& filename, int blockID); 106 | 107 | File &getFile(const string& filename); 108 | 109 | Block &findBlock(const string& filename, int blockID) const; 110 | 111 | int LRUNum; 112 | int fileCnt; 113 | 114 | Block *blockHandle; 115 | }; 116 | 117 | #endif //MINISQL_BUFFERMANAGER_H 118 | -------------------------------------------------------------------------------- /db/BufferManager/Readme.md: -------------------------------------------------------------------------------- 1 | ## Buffer Manager module design report 2 | 3 | writer: 3190104509 Gao Weiyuan 4 | 5 | ### Overview 6 | 7 | `BufferManager` constructs and manages buffers in memory. Other modules, like `RecordManager` and `IndexManager`, asks for the data from `BufferManager` instead of directly getting them from the file system. It smooth the low speed of disk I/O and dramatically improve the performance of database. 8 | 9 | Each file is divided into many blocks and the size of each block is 4K. While the program is running, `BufferManager` would read some blocks into memory when needed and offer them to other modules. 10 | 11 | ### Module Design 12 | 13 | `RecordManager` and `IndexManager` request data from `BufferManager`. When a request occur, `BufferManager` firstly check if the data exist in buffer. If exist, return the data directly. If not, `BufferManager` will read the data from disk, then check if the buffer has free space, if so add the new block to the buffer. If not a block will be choosen to be replaced using LRU algorithm. 14 | 15 | LRU(Least recently used), always choose the least recently used blcok. There is a LRU count variable in each block. Whenever the block is used, the LRU count increases by 1. Then the block that has the least LRU count is the block to be replaced. 16 | 17 | In buffer, I maintain the following two structures: 18 | 19 | - File Blocks 20 | 21 | This structure contains blocks store useful data. The FileHandle points to a file linked list, each node represents a file. And each node in file linked list points to a batch of block. When `BufferManager` handle a request, it firstly find in file linked list. If the file is not found, it will create a new node and open the file. Then it search the blocks under the file node. If the block is in buffer, it will be returned directly. If not, the manager will fetch a block in free block linked list and add it to file blocks. 22 | 23 | - Free Blocks 24 | 25 | Free block is a linked list headed by BlockHandle which stores blocks not used. When a file is closed or be replaced, it will reduce a lot of free blocks, these blocks will be added in to free blocks linked list. When we need new blocks, we can fetch blocked in this linked list. 26 | 27 | ### Implementation 28 | 29 | 1. Constant defined 30 | There are several parameters defined as follow: 31 | 32 | ```c++ 33 | const int BlockSize = 4096; 34 | const int MaxBlocks = 40; 35 | const int MaxFiles = 6; 36 | ``` 37 | 38 | BlockSize defines each block's size. MaxBlocks defines the maximum of the number of blocks. MaxFiles defines the maximum of the number of opening files. 39 | 40 | 2. data structure 41 | 42 | - struct File 43 | 44 | ```c++ 45 | struct File { 46 | int type; //0:db file 1:table file 2: index file 47 | 48 | string filename; //the name of the file 49 | int blockCnt; 50 | int totBlockCnt; 51 | File *next; //the pointer points to the next file 52 | Block *firstBlock; 53 | 54 | File(const string& _filename); 55 | 56 | }; 57 | ``` 58 | 59 | 60 | 61 | - struct Block 62 | 63 | ```c++ 64 | struct Block { 65 | int blockID; 66 | bool dirty; //whether the block been changed 67 | Block *next; 68 | File *file; 69 | int usedSize; 70 | int LRUCount; 71 | int lock; //prevent the block from replacing 72 | char *blockContent; 73 | 74 | Block(); 75 | Block(bool isNull); 76 | ~Block() = default; 77 | }; 78 | ``` 79 | 80 | 81 | 82 | 3. Major function introduction 83 | 84 | - Get Block 85 | 86 | ```c++ 87 | Block &getBlock(const string& fileName, int blockID); 88 | ``` 89 | 90 | filename: name of the file 91 | 92 | blockID: ID of requested block. 93 | 94 | function: Place a specified block in buffer. It read data from file system in case of missing block in memory and use the LRU algorithm as the block-replacement-strategy. If the block is not exist, buffer manager will allocate a new block automatically. 95 | 96 | - Set Dirty 97 | 98 | ```c++ 99 | void setDirty(const string &filename, unsigned int blockID); 100 | ``` 101 | 102 | filename: name of the file 103 | 104 | blockID: ID of the block. 105 | 106 | Set the dirty-bit in block, which indicates the data in block have been modified and it shoud write back first before replacing. 107 | 108 | - Unlock 109 | 110 | ```c++ 111 | void unlock(const string& filename, unsigned int blockID); 112 | ``` 113 | 114 | filename: name of the file 115 | 116 | blockID: ID of the block. function: Unlock a block, allowing it to be replaced by other block. 117 | 118 | - Create File 119 | 120 | ```c++ 121 | static void createFile(const string& in); 122 | ``` 123 | 124 | in: file name 125 | 126 | function: Creat a file in file system, which is initially blank 127 | 128 | - Remove File 129 | 130 | ```c++ 131 | void removeFile(const string& filename); 132 | ``` 133 | 134 | filename: name of the file needed to be remove. 135 | 136 | function: Remove a file from file system. 137 | 138 | - Get Block Cnt 139 | 140 | ```c++ 141 | int getBlockCnt(const string& filename); 142 | ``` 143 | 144 | filename: name of the file. 145 | 146 | function: Get the number of total blocks of a specified file. 147 | 148 | - Close All File 149 | 150 | ```c++ 151 | void closeAllFile(); 152 | ``` 153 | 154 | function: Save all data and close all files, called when quitting the process. 155 | 156 | -------------------------------------------------------------------------------- /db/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(minisql) 3 | 4 | set(CMAKE_CXX_STANDARD 20) 5 | 6 | set(INTERPRETER 7 | interpreter/interpreter.h 8 | interpreter/interpreter.cpp 9 | interpreter/readline.c) 10 | 11 | set(API 12 | api/api.h 13 | api/api.cpp 14 | api/utils.cpp 15 | api/utils.h 16 | ) 17 | 18 | set(EXCEPTION 19 | utils/exception.cpp 20 | utils/exception.h 21 | ) 22 | 23 | set(DEPRECATED_INTERPRETER 24 | interpreter/lex.yy.cc 25 | interpreter/parser.tab.hh 26 | interpreter/parser.tab.cc 27 | utils/exception.cpp utils/exception.h 28 | ) 29 | 30 | set(CATALOG_MANAGER 31 | CatalogManager/CatalogManager.h 32 | CatalogManager/CatalogManager.cpp 33 | ) 34 | 35 | set(BUFFER_MANAGER 36 | BufferManager/BufferManager.h 37 | BufferManager/BufferManager.cpp 38 | ) 39 | 40 | set(RECORD_MANAGER 41 | RecordManager/RecordManager.h 42 | RecordManager/RecordManager.cpp 43 | ) 44 | 45 | set(INDEX_MANAGER 46 | IndexManager/BPInclude.h 47 | IndexManager/BPTree.h 48 | IndexManager/BPTreeNode.h 49 | IndexManager/BPTree.cpp 50 | IndexManager/IndexManager.h 51 | IndexManager/IndexManager.cpp 52 | IndexManager/BPTreeNode.cpp) 53 | 54 | # add_definitions(-DDETACH_INDEX_MANAGER) 55 | 56 | find_package(BISON 3.0) 57 | find_package(FLEX) 58 | if(BISON_FOUND AND FLEX_FOUND) 59 | file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/interpreter) 60 | BISON_TARGET(MyParser interpreter/parser.y ${CMAKE_CURRENT_BINARY_DIR}/interpreter/parser.tab.cc) 61 | FLEX_TARGET(MyScanner interpreter/tokenizer.l ${CMAKE_CURRENT_BINARY_DIR}/interpreter/lex.yy.cc) 62 | ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser) 63 | include_directories(interpreter) 64 | add_library(${PROJECT_NAME} STATIC macro.h external_main/external_main.cpp 65 | ${INTERPRETER} ${API} ${EXCEPTION} 66 | ${BISON_MyParser_OUTPUTS} ${FLEX_MyScanner_OUTPUTS} 67 | ${BUFFER_MANAGER} ${RECORD_MANAGER} ${CATALOG_MANAGER} ${INDEX_MANAGER} 68 | utils/utils.cpp utils/utils.h) 69 | add_executable(${PROJECT_NAME}_local main.cpp) 70 | target_link_libraries(${PROJECT_NAME}_local ${PROJECT_NAME}) 71 | else() 72 | message(FATAL_ERROR "GNU Bison(version >= 3.0) or Flex not found.") 73 | endif() 74 | 75 | file(COPY 76 | ${CMAKE_CURRENT_SOURCE_DIR}/go-svc/main.go 77 | ${CMAKE_CURRENT_SOURCE_DIR}/go-svc/placeholder.cpp 78 | ${CMAKE_CURRENT_SOURCE_DIR}/go.mod 79 | ${CMAKE_CURRENT_SOURCE_DIR}/go.sum 80 | DESTINATION 81 | ${CMAKE_CURRENT_BINARY_DIR}/go-svc 82 | ) 83 | file(COPY 84 | ${CMAKE_CURRENT_SOURCE_DIR}/external_main/external_main.h 85 | DESTINATION 86 | ${CMAKE_CURRENT_BINARY_DIR}/external_main 87 | ) 88 | 89 | add_custom_command( 90 | TARGET ${PROJECT_NAME} 91 | POST_BUILD 92 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/go-svc 93 | COMMAND go build -o ../${PROJECT_NAME}_region 94 | COMMENT "Building go source files" 95 | ) 96 | 97 | # if(LIB_READLINE) 98 | # message(STATUS "Found READLINE: ${LIB_READLINE}") 99 | # add_definitions(-DREADLINE_FOUND) 100 | # target_link_libraries(MiniSQL ${LIB_READLINE}) 101 | # else() 102 | # message(WARNING "Library \"readline\" not found. \ 103 | # This program will use C++ standard library instead... (not recommended)") 104 | # endif() 105 | -------------------------------------------------------------------------------- /db/CatalogManager/CatalogManager.cpp: -------------------------------------------------------------------------------- 1 | #include "CatalogManager.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "../IndexManager/IndexManager.h" 9 | #include "../utils/utils.h" 10 | 11 | using namespace std; 12 | 13 | extern IndexManager idx_mgt; 14 | 15 | 16 | void CatalogManager::CreateTable(const std::string &table_name, 17 | const std::vector &schema_list, 18 | const std::string &primary_key_name 19 | ) 20 | { 21 | macro::table tab(table_name, schema_list.size()); 22 | unsigned long long len = 0; 23 | 24 | 25 | for (auto &schema: schema_list){ 26 | auto elem_type = schema.type; 27 | len += elem_type.size(); 28 | tab.attribute_names.push_back(schema.name); 29 | if (schema.name == primary_key_name){ 30 | elem_type.primary = true; 31 | } 32 | if (schema.unique){ 33 | elem_type.unique = true; 34 | } 35 | if (elem_type.type == value_type::CHAR){ 36 | elem_type.length = schema.type.length; 37 | } 38 | if (elem_type.primary){ 39 | auto default_index = generate_table_pk_name(table_name); 40 | CreateIndex(tab, primary_key_name, default_index); 41 | } 42 | 43 | tab.attribute_type.push_back(elem_type); 44 | } 45 | 46 | tab.record_len = len; 47 | tab.record_cnt = 0; 48 | 49 | 50 | tables.push_back(tab); 51 | } 52 | 53 | CatalogManager::CatalogManager() 54 | :tables(std::vector()) 55 | { 56 | this->LoadFromFile(); 57 | } 58 | 59 | CatalogManager::~CatalogManager() 60 | { 61 | Flush(); 62 | } 63 | 64 | void CatalogManager::Flush() 65 | { 66 | std::ofstream ofs(tables_info); 67 | 68 | ofs << tables.size() << std::endl; 69 | 70 | for (auto &tb: tables){ 71 | ofs << tb.name << std::endl << tb.record_cnt << std::endl; 72 | std::ofstream tab_ofs(tb.name + ".catalog"); 73 | 74 | tab_ofs << tb.attribute_names.size() << std::endl; 75 | for(int i = 0; i < tb.attribute_names.size(); ++i){ 76 | tab_ofs << tb.attribute_names[i] << std::endl; 77 | const auto &attr = tb.attribute_type[i]; 78 | switch (attr.type){ 79 | case value_type::INT: 80 | tab_ofs << "int" << std::endl; 81 | tab_ofs << 0 << std::endl; 82 | break; 83 | case value_type::FLOAT: 84 | tab_ofs << "float" << std::endl; 85 | tab_ofs << 0 << std::endl; 86 | break; 87 | case value_type::CHAR: 88 | tab_ofs << "char" << std::endl; 89 | tab_ofs << std::to_string(attr.length) << std::endl; 90 | break; 91 | } 92 | tab_ofs << ((attr.primary) ? 1 : 0) << std::endl; 93 | tab_ofs << ((attr.unique) ? 1 : 0) << std::endl; 94 | 95 | // Index info 96 | bool ifind = false; 97 | for(auto ind = tb.index.begin(); ind != tb.index.end(); ++ind){ 98 | if(ind->first == tb.attribute_names[i]){ 99 | tab_ofs << 1 << std::endl << ind->second << std::endl; 100 | ifind = true; 101 | break; 102 | } 103 | } 104 | 105 | if (!ifind){ 106 | tab_ofs << 0 << std::endl << "-" << std::endl; 107 | } 108 | } 109 | tab_ofs.close(); 110 | } 111 | ofs.close(); 112 | } 113 | 114 | void CatalogManager::LoadFromFile() 115 | { 116 | std::ifstream ifs(tables_info); 117 | if(!ifs){ 118 | ifs.open(tables_info); 119 | return; 120 | } 121 | 122 | int table_size = 0; 123 | ifs >> table_size; 124 | 125 | for (int i = 0; i < table_size; ++i){ 126 | macro::table tb; 127 | ifs >> tb.name >> tb.record_cnt; 128 | std::string tabfile_name = tb.name + ".catalog"; 129 | std::ifstream tab_ifs(tabfile_name); 130 | 131 | unsigned long long record_len = 0, attr_counts = 0; 132 | 133 | tab_ifs >> attr_counts; 134 | for (int attr_i = 0; attr_i < attr_counts; ++attr_i){ 135 | 136 | std::string attr_name, type_name, index_name; 137 | tab_ifs >> attr_name; 138 | tb.attribute_names.push_back(attr_name); 139 | 140 | int type_size; 141 | sql_value_type type; 142 | tab_ifs >> type_name >> type_size; 143 | 144 | if (type_name == "int"){ 145 | type.type = value_type::INT; 146 | } 147 | else if (type_name == "char"){ 148 | type.type = value_type::CHAR; 149 | } 150 | else if (type_name == "float"){ 151 | type.type = value_type::FLOAT; 152 | } 153 | type.length = type_size; 154 | 155 | short ifpri, ifind, ifuniq; 156 | tab_ifs >> ifpri >> ifuniq; 157 | tab_ifs >> ifind >> index_name; 158 | 159 | record_len += type.size(); 160 | type.primary = (ifpri) ? 1 : 0; 161 | type.unique = (ifuniq) ? 1 : 0; 162 | tb.attribute_type.push_back(type); 163 | 164 | if (ifind == 1){ 165 | CreateIndex(tb, attr_name, index_name); 166 | } 167 | else{ 168 | if(ifind != 0 || index_name != "-"){ 169 | throw sql_exception(604, "catalog manager", "catalog file crached on index \'" + index_name + "\'"); 170 | } 171 | } 172 | 173 | } 174 | tb.attribute_cnt = attr_counts; 175 | tb.record_len = record_len; 176 | tables.push_back(tb); 177 | 178 | 179 | } 180 | } 181 | 182 | bool CatalogManager::TableExist(const std::string &query_name) const 183 | { 184 | for (const auto & table : tables){ 185 | if(table.name == query_name){ 186 | return true; 187 | } 188 | } 189 | return false; 190 | } 191 | 192 | macro::table &CatalogManager::GetTable(const std::string &query_name) 193 | { 194 | for (auto & table : tables){ 195 | if(table.name == query_name){ 196 | return table; 197 | } 198 | } 199 | throw sql_exception(603, "catalog manager", "cannot find table \'" + query_name + "\'"); 200 | } 201 | 202 | 203 | bool CatalogManager::DropTableByName(const std::string &table_name) 204 | { 205 | 206 | std::vector::iterator table_to_drop; 207 | for ( auto table = tables.begin(); table != tables.end(); ++table){ 208 | if((*table).name == table_name){ 209 | table_to_drop = table; 210 | } 211 | } 212 | std::string tab_file; 213 | tab_file = table_name + ".catalog"; 214 | std::ifstream ifs(tab_file, ios::in); 215 | if(ifs.is_open()){ 216 | remove(const_cast(tab_file.c_str())); 217 | } 218 | 219 | 220 | if(table_to_drop != tables.end()){ 221 | tables.erase(table_to_drop); 222 | return true; 223 | } 224 | return false; 225 | } 226 | 227 | macro::table &CatalogManager::GetTableWithIndex(const std::string &index_name) 228 | { 229 | for (auto & table : tables){ 230 | for (auto i = table.index.begin(); i != table.index.end(); ++i){ 231 | if (i->second == index_name){ 232 | return table; 233 | } 234 | } 235 | } 236 | throw sql_exception(600, "catalog manager", "cannot find index \'" + index_name + "\'"); 237 | } 238 | /* 239 | bool CatalogManager::IsIndexExist(const std::string &index_name) 240 | { 241 | for (auto & table : tables){ 242 | for (auto & i : table.index){ 243 | if (i.second == index_name){ 244 | return true; 245 | } 246 | } 247 | } 248 | return false; 249 | } 250 | */ 251 | 252 | bool CatalogManager::CreateIndex(const std::string &table_name, const std::string &attr_name, const std::string &index_name) 253 | { 254 | if(!TableExist(table_name)){ 255 | return false; 256 | } 257 | macro::table &tab = GetTable(table_name); 258 | tab.index.emplace_back(attr_name, index_name); 259 | const auto [iter, success] = indexes.insert({index_name, table_name}); 260 | if(!success){ 261 | throw sql_exception(601, "catalog manager", "repetition of index name \'" + index_name + "\'"); 262 | } 263 | return true; 264 | } 265 | 266 | bool CatalogManager::CreateIndex(macro::table &table, const std::string &attr_name, const std::string &index_name) 267 | { 268 | table.index.emplace_back(attr_name, index_name); 269 | const auto [iter, success] = indexes.insert({index_name, table.name}); 270 | if(!success){ 271 | throw sql_exception(601, "catalog manager", "repetition of index name \'" + index_name + "\'"); 272 | } 273 | return true; 274 | } 275 | 276 | bool CatalogManager::DropIndex(const std::string &table_name, const std::string &attr_name) 277 | { 278 | if(!TableExist(table_name)){ 279 | return false; 280 | } 281 | macro::table &tab = GetTable(table_name); 282 | for (auto iter = tab.index.begin(); iter != tab.index.end(); iter++){ 283 | if (iter->first == attr_name){ 284 | tab.index.erase(iter); 285 | auto index_iter = indexes.find(iter->second); 286 | if(index_iter != indexes.end()){ 287 | indexes.erase(index_iter); 288 | } 289 | return true; 290 | } 291 | } 292 | return false; 293 | 294 | } 295 | 296 | // create index asd on foo(bar); 297 | bool CatalogManager::DropIndex(const std::string &index_name) 298 | { 299 | std::string attr_name; 300 | std::string table_name; 301 | 302 | try { 303 | table_name = indexes.at(index_name); 304 | } catch (const out_of_range &e){ 305 | throw sql_exception(602, "catalog manager", "index not found"); 306 | } 307 | 308 | auto &tab = GetTable(table_name); 309 | for (const auto &i : tab.index) { 310 | if (i.second == index_name) { 311 | attr_name = i.first; 312 | } 313 | } 314 | if (attr_name.empty()) { 315 | throw sql_exception(602, "catalog manager", "index not found"); 316 | } 317 | 318 | for (auto iter = tab.index.begin(); iter != tab.index.end(); iter++){ 319 | if (iter->first == attr_name){ 320 | tab.index.erase(iter); 321 | auto index_iter = indexes.find(iter->second); 322 | if(index_iter != indexes.end()){ 323 | indexes.erase(index_iter); 324 | } 325 | return true; 326 | } 327 | } 328 | return false; 329 | } 330 | 331 | std::vector CatalogManager::get_table_names() { 332 | auto vec = std::vector(); 333 | vec.reserve(tables.size()); 334 | for (auto x : tables) { 335 | vec.push_back(x.name); 336 | } 337 | return vec; 338 | } 339 | -------------------------------------------------------------------------------- /db/CatalogManager/CatalogManager.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef MINISQL_CATALOGMANAGER_H 3 | #define MINISQL_CATALOGMANAGER_H 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../macro.h" 13 | 14 | //using namespace MINISQL_BASE; 15 | 16 | class CatalogManager 17 | { 18 | public: 19 | void CreateTable(const std::string &table_name, 20 | const std::vector &schema_list, 21 | const std::string &primary_key_name 22 | ); 23 | 24 | bool TableExist(const std::string &table_name) const; 25 | 26 | macro::table &GetTable(const std::string &table_name); 27 | 28 | macro::table &GetTableWithIndex(const std::string &index_name); 29 | 30 | // bool IsIndexExist(const std::string &index_name); 31 | 32 | bool DropTableByName(const std::string &table_name); 33 | 34 | CatalogManager(); 35 | 36 | ~CatalogManager(); 37 | 38 | void Flush() ; 39 | 40 | bool isValid() const 41 | { return validFlag; } 42 | 43 | mutable std::map kv; 44 | 45 | void LoadFromFile(); 46 | 47 | bool CreateIndex(const std::string &table_name, const std::string &attr_name, const std::string &index_name); 48 | 49 | bool CreateIndex(macro::table &table, const std::string &attr_name, const std::string &index_name); 50 | 51 | bool DropIndex(const std::string &table_name, const std::string &attr_name); 52 | 53 | bool DropIndex(const std::string &index_name); 54 | 55 | std::vector get_table_names(); 56 | 57 | 58 | private: 59 | std::vector tables; 60 | 61 | std::map indexes; // 62 | 63 | const std::string tables_info = "tables.meta"; 64 | 65 | bool validFlag = true; 66 | }; 67 | 68 | #endif //MINISQL_CATALOGMANAGER_H 69 | -------------------------------------------------------------------------------- /db/CatalogManager/README.md: -------------------------------------------------------------------------------- 1 | # CatalogManager 2 | 3 | The `CatalogManager` is responsible for the metadata in our DBMS. It mainly maintains the schema which contains definitions of tables, columns with attribute names, indexes and primary keys. It provides API module with interface that can deal with the access to the metadata above. 4 | 5 | The module exposes the following public methods: 6 | 7 | - `CreateTable` 8 | To create a schema of table, the function receives a table name, list of attributes with their types and a primary key name if possible, then it will create the catalog of the table in memory. 9 | 10 | - `Flush` 11 | This function is responsible for the persistence of meta info of tables, which contains the number of tables altogether, the name and record count of each table, the attributes with names, whether unique or primary info and indexes with index names. 12 | 13 | - `LoadFromFile` 14 | 15 | Read in schema info from external storage with the same format into the memory, this function is called during the initialization stage of the whole system. 16 | 17 | - `TableExist` 18 | To check whether the table with a certain name exists in the memory. 19 | 20 | - `GetTable` 21 | Search in memory to find the specific table and returns its reference. 22 | 23 | - `DropTableByName` 24 | Drop a table in memory according to its name. 25 | 26 | - `CreateIndex` 27 | Create an index on specific attribute with index name. 28 | 29 | - `DropIndex` 30 | Drop an index of an attribute according to the index name. 31 | 32 | In order to reduce the coupling between the modules, the Catalog module uses the form of direct access to disk files without going through the Buffer Manager. -------------------------------------------------------------------------------- /db/IndexManager/BPInclude.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | using std::vector; 9 | using std::queue; 10 | 11 | using ELEMENTTYPE = uint32_t; -------------------------------------------------------------------------------- /db/IndexManager/BPTree.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../utils/exception.h" 4 | #include "BPInclude.h" 5 | #include "BPTreeNode.h" 6 | 7 | #include 8 | #include 9 | 10 | using std::string; 11 | 12 | class BPTree { 13 | public: 14 | void insert(uint32_t idx_pos, 15 | const macro::table& table, 16 | ELEMENTTYPE rec_id, 17 | const sql_value& target); 18 | 19 | ELEMENTTYPE search(uint32_t idx_pos, 20 | const macro::table& table, 21 | const sql_value& target) const; 22 | 23 | void remove(uint32_t idx_pos, 24 | const macro::table& table, 25 | const sql_value& target, 26 | std::unordered_map umap); 27 | 28 | void write_file(string filename, string treename); 29 | 30 | ELEMENTTYPE searchHead() const; 31 | 32 | ELEMENTTYPE searchNext(uint32_t idx_pos, 33 | const macro::table& table, 34 | const sql_value& target) const; 35 | 36 | BPTree() = default; 37 | 38 | ~BPTree(); 39 | 40 | BPTree(uint32_t max_size, uint32_t deg); 41 | 42 | BPTree(string treestr, uint32_t max_size, string& treename); 43 | 44 | private: 45 | BPTreeNode* root = nullptr; 46 | uint32_t degree; 47 | uint32_t size; 48 | 49 | BPTreeNode* pool = nullptr; 50 | queue deleted_id; 51 | 52 | BPTreeNode* new_node(); 53 | 54 | void delete_node(BPTreeNode* node); 55 | }; 56 | -------------------------------------------------------------------------------- /db/IndexManager/BPTreeNode.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ralxyz on 6/22/21. 3 | // 4 | 5 | #include "../RecordManager/RecordManager.h" 6 | #include "BPTree.h" 7 | 8 | extern RecordManager rec_mgt; 9 | 10 | sql_value find_idx_val(ELEMENTTYPE rec_id, 11 | uint32_t idx_pos, 12 | const macro::table& table) { 13 | return rec_mgt.getRecord(table, rec_id).element.at(idx_pos); 14 | } 15 | 16 | sql_value find_idx_val(ELEMENTTYPE rec_id, 17 | uint32_t idx_pos, 18 | const macro::table& table, 19 | std::unordered_map umap) { 20 | auto res = umap.find(rec_id); 21 | return res == umap.end() ? find_idx_val(rec_id, idx_pos, table) 22 | : res->second.element.at(idx_pos); 23 | } 24 | 25 | uint32_t BPTreeNode::binary_search(uint32_t idx_pos, 26 | const macro::table& table, 27 | const sql_value& target) const { 28 | int l = 0, r = key.size() - 1; 29 | while (l <= r) { 30 | int mid = (l + r) / 2; 31 | if (find_idx_val(key.at(mid), idx_pos, table) <= target) 32 | l = mid + 1; 33 | else 34 | r = mid - 1; 35 | } 36 | return l; 37 | } 38 | 39 | uint32_t BPTreeNode::binary_search( 40 | uint32_t idx_pos, 41 | const macro::table& table, 42 | const sql_value& target, 43 | std::unordered_map umap) const { 44 | int l = 0, r = key.size() - 1; 45 | while (l <= r) { 46 | int mid = (l + r) / 2; 47 | if (find_idx_val(key.at(mid), idx_pos, table, umap) <= target) 48 | l = mid + 1; 49 | else 50 | r = mid - 1; 51 | } 52 | return l; 53 | } -------------------------------------------------------------------------------- /db/IndexManager/BPTreeNode.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../macro.h" 4 | #include "BPInclude.h" 5 | 6 | sql_value find_idx_val(ELEMENTTYPE rec_id, 7 | uint32_t idx_pos, 8 | const macro::table& table); 9 | 10 | sql_value find_idx_val(ELEMENTTYPE rec_id, 11 | uint32_t idx_pos, 12 | const macro::table& table, 13 | std::unordered_map umap); 14 | 15 | class BPTreeNode { 16 | public: 17 | BPTreeNode *fa, *nxt; 18 | vector ch; 19 | vector key; 20 | 21 | // for leaf node: insert pos 22 | // for non-leaf node: which child is in 23 | uint32_t binary_search(uint32_t idx_pos, 24 | const macro::table& table, 25 | const sql_value& target) const; 26 | 27 | uint32_t binary_search( 28 | uint32_t idx_pos, 29 | const macro::table& table, 30 | const sql_value& target, 31 | std::unordered_map umap) const; 32 | }; 33 | -------------------------------------------------------------------------------- /db/IndexManager/IndexManager.cpp: -------------------------------------------------------------------------------- 1 | #include "IndexManager.h" 2 | 3 | #include 4 | 5 | IndexManager::IndexManager() { 6 | this->load(); 7 | } 8 | 9 | IndexManager::~IndexManager() { 10 | // TODO: destruct function 11 | } 12 | 13 | void IndexManager::create(const string& treename) { 14 | index_manager.emplace(treename, std::make_unique(TREE_SIZE, TREE_DEGREE)); 15 | } 16 | 17 | void IndexManager::create(const string& treename, 18 | const vector>& indexs, 19 | uint32_t idx_pos, 20 | const macro::table table) { 21 | auto idxtree = std::make_unique(TREE_SIZE, TREE_DEGREE); 22 | 23 | for (auto& x : indexs) 24 | 25 | idxtree->insert(idx_pos, table, x.first, x.second); 26 | index_manager.emplace(treename, std::move(idxtree)); 27 | } 28 | 29 | void IndexManager::drop(const string& treename) { 30 | index_manager.erase(treename); 31 | } 32 | 33 | ELEMENTTYPE IndexManager::search(const string& treename, 34 | const sql_value& val, 35 | uint32_t idx_pos, 36 | const macro::table& table) const { 37 | return index_manager.find(treename)->second->search(idx_pos, table, val); 38 | } 39 | 40 | ELEMENTTYPE IndexManager::searchHead(const string& treename, 41 | const sql_value& val) const { 42 | return index_manager.find(treename)->second->searchHead(); 43 | } 44 | 45 | ELEMENTTYPE IndexManager::searchNext(const string& treename, 46 | const sql_value& val, 47 | uint32_t idx_pos, 48 | const macro::table& table) const { 49 | return index_manager.find(treename)->second->searchNext(idx_pos, table, val); 50 | } 51 | 52 | void IndexManager::insert(const string& treename, 53 | const sql_value& val, 54 | uint32_t idx_pos, 55 | const macro::table& table, 56 | ELEMENTTYPE new_key) { 57 | index_manager.find(treename)->second->insert(idx_pos, table, new_key, val); 58 | } 59 | 60 | void IndexManager::remove(const string& treename, 61 | const sql_value& val, 62 | uint32_t idx_pos, 63 | const macro::table& table, 64 | std::unordered_map umap) { 65 | index_manager.find(treename)->second->remove(idx_pos, table, val, umap); 66 | } 67 | 68 | void IndexManager::save() { 69 | std::ofstream f(index_file); 70 | f << index_manager.size() << std::endl; 71 | for (auto& x : index_manager) { 72 | x.second->write_file(index_file, x.first); 73 | } 74 | } 75 | 76 | void IndexManager::load() { 77 | std::ifstream f(index_file); 78 | if (!f) 79 | return; 80 | int map_sz; 81 | string map_sz_str; 82 | std::getline(f, map_sz_str); 83 | map_sz = std::stoi(map_sz_str); 84 | for (int i = 0; i < map_sz; ++i) { 85 | string treestr, name; 86 | std::getline(f, treestr); 87 | auto tree = std::make_unique(treestr, TREE_SIZE, name); 88 | index_manager.emplace(name, std::move(tree)); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /db/IndexManager/IndexManager.h: -------------------------------------------------------------------------------- 1 | #include "../macro.h" 2 | #include "BPTree.h" 3 | 4 | #include 5 | #include 6 | #include 7 | using std::make_pair; 8 | using std::string; 9 | 10 | const string index_file = "trees.ind"; 11 | const int TREE_SIZE = 666; 12 | const int TREE_DEGREE = 4; 13 | 14 | using std::get; 15 | using std::make_tuple; 16 | 17 | class IndexManager { 18 | public: 19 | IndexManager(); 20 | ~IndexManager(); 21 | 22 | void create(const string& treename); 23 | void create( 24 | const string& treename, 25 | const vector>& indexs, 26 | uint32_t idx_pos, 27 | const macro::table table); 28 | void drop(const string& treename); 29 | 30 | ELEMENTTYPE search(const string& treename, 31 | const sql_value& val, 32 | uint32_t idx_pos, 33 | const macro::table& table) const; 34 | ELEMENTTYPE searchHead(const string& treename, const sql_value& val) const; 35 | ELEMENTTYPE searchNext(const string& treename, 36 | const sql_value& val, 37 | uint32_t idx_pos, 38 | const macro::table& table) const; 39 | 40 | void insert(const string& treename, 41 | const sql_value& val, 42 | uint32_t idx_pos, 43 | const macro::table& table, 44 | ELEMENTTYPE new_key); 45 | void remove(const string& treename, 46 | const sql_value& val, 47 | uint32_t idx_pos, 48 | const macro::table& table, 49 | std::unordered_map umap); 50 | 51 | void save(); 52 | void load(); 53 | 54 | private: 55 | std::unordered_map> index_manager; 56 | }; 57 | -------------------------------------------------------------------------------- /db/IndexManager/Readme.md: -------------------------------------------------------------------------------- 1 | # IndexManager 2 | 3 | As what is just mentioned in its name, the module `IndexManager` is a class that works for the management of the index. 4 | 5 | The index manager supports the following functions, which are listed as `public` member methods: 6 | 7 | - create 8 | The `create` method supports two types of index creation: 9 | 1. Create an empty index on a column of table, for example when called at `create table` 10 | 2. Create an index on existing values 11 | - drop 12 | Remove an index from index manager 13 | - insert 14 | Insert a single element to a certain index 15 | - search 16 | Search for a single element in a certain index, the return value is position of the target row in table (in `int`) 17 | - remove 18 | Remove a single element from a certain index 19 | - save 20 | Save all the indexes to local index file, should be called when quitting the process 21 | - load 22 | Load the indexes from local file to memory, should be called when the process starts 23 | 24 | Each of the index is individually managed by a customize B+ tree, and we use an unordered_map (the only private member in the class) to relate the index name with its content. 25 | 26 | In order to save the space in B+ tree storage, the keys stored in B+ tree is actually the row number. And we use an external function provided by `RecordManager` to get the real value for comparing when operating the B+ tree. 27 | 28 | The B+ tree currently has three main functions: insert, delete and search. Some other values and method are marked public, but they are _unused_ and _left for future accomplishment_. 29 | 30 | The details of implementation will be illustrate in following chapters. 31 | 32 | ## Interface 33 | 34 | The module `IndexManager` contains two parts: the manager layer and the B+ tree layer. 35 | 36 | The manager layer's interfaces are: 37 | 38 | - constructor: nothing else other than default constructor is needed 39 | 40 | ```cpp 41 | IndexManager() = default; 42 | ``` 43 | 44 | - destructor: nothing extra needs to be destroyed 45 | 46 | ```cpp 47 | ~IndexManager(); 48 | ``` 49 | 50 | - create index: there are two types: 51 | 52 | 1. build an empty index by name 53 | 54 | ```cpp 55 | void create(const string& treename); 56 | ``` 57 | 58 | 2. build an index on existing elements 59 | 60 | ```cpp 61 | void create(const string& treename, 62 | const vector>& indexs, 63 | uint32_t idx_pos, 64 | const macro::table table); 65 | ``` 66 | 67 | Explanation: 68 | 69 | - the vector `indexs` stores those exising value 70 | - `idx_pos` marks which column is the index in the table (the same below) 71 | 72 | - drop index: just drop by name 73 | 74 | ```cpp 75 | void drop(const string& treename); 76 | ``` 77 | 78 | - search: search the position of value `val` 79 | 80 | ```cpp 81 | search(const string& treename, 82 | const sql_value& val, 83 | uint32_t idx_pos, 84 | const macro::table& table) const; 85 | ``` 86 | 87 | - insert: insert a value `val` at `new_key` 88 | 89 | ```cpp 90 | void insert(const string& treename, 91 | const sql_value& val, 92 | uint32_t idx_pos, 93 | const macro::table& table, 94 | ELEMENTTYPE new_key); 95 | ``` 96 | 97 | - remove: delete a value `val` 98 | 99 | ```cpp 100 | void remove(const string& treename, 101 | const sql_value& val, 102 | uint32_t idx_pos, 103 | const macro::table& table, 104 | std::unordered_map umap); 105 | ``` 106 | 107 | - save & load: file operation 108 | 109 | ```cpp 110 | void save(); 111 | void load(); 112 | ``` 113 | 114 | - `searchNext` and `searchHead` are not currently used, so we won't give the definition here 115 | 116 | The B+ tree layer's interfaces are: 117 | 118 | - constructor: 119 | 120 | 1. The default constructor 121 | 122 | ```cpp 123 | BPTree() = default; 124 | ``` 125 | 126 | 2. Init an empty tree by max size and degree 127 | 128 | ```cpp 129 | BPTree(uint32_t max_size, uint32_t deg); 130 | ``` 131 | 132 | 3. Init a tree from a string (should be read from file) 133 | 134 | ```cpp 135 | BPTree(string treestr, uint32_t max_size, string& treename); 136 | ``` 137 | 138 | - destructor: we just need to delete the pool we apply 139 | 140 | ```cpp 141 | ~BPTree(); 142 | ``` 143 | 144 | - insert 145 | 146 | ```cpp 147 | void insert(uint32_t idx_pos, 148 | const macro::table& table, 149 | ELEMENTTYPE rec_id, 150 | const sql_value& target); 151 | ``` 152 | 153 | - search 154 | 155 | ```cpp 156 | ELEMENTTYPE search(uint32_t idx_pos, 157 | const macro::table& table, 158 | const sql_value& target) const; 159 | ``` 160 | 161 | - remove 162 | 163 | ``` 164 | void remove(uint32_t idx_pos, 165 | const macro::table& table, 166 | const sql_value& target, 167 | std::unordered_map umap); 168 | ``` 169 | 170 | - write a tree named `treename` to file named `filename` 171 | 172 | ```cpp 173 | void write_file(string filename, string treename); 174 | ``` 175 | 176 | ## Implementation Detail 177 | 178 | For index manager, it is almost a wrapper for B+ tree 179 | 180 | We just use a map to relate the B+ tree with the index name 181 | 182 | And when we do operations on index manager, we actually call the api provided by B+ trees 183 | 184 | ## B+ tree details 185 | 186 | ### B+ tree nodes 187 | 188 | For each node in B+ trees, we use a class `BPTreeNode` to represent it 189 | 190 | ```cpp 191 | class BPTreeNode { 192 | public: 193 | BPTreeNode *fa, *nxt; 194 | vector ch; 195 | vector key; 196 | uint32_t binary_search(uint32_t idx_pos, 197 | const macro::table& table, 198 | const sql_value& target) const; 199 | uint32_t binary_search( 200 | uint32_t idx_pos, 201 | const macro::table& table, 202 | const sql_value& target, 203 | std::unordered_map umap) const; 204 | }; 205 | ``` 206 | 207 | The members are: 208 | 209 | - `fa`: the father of the node 210 | - `ch`: the children of the node 211 | - `nxt`: the next node in the bottom layer, should remain `nullptr` for upper ones 212 | - `key`: the marks of the node 213 | 214 | For those leaf nodes, `ch` are always `nullptr`, and the number is the same as `key`. 215 | For those non-leaf nodes, the `ch` of number should be one more than `key`. 216 | 217 | The `binary_search` method is use to find a target's position, for leaf node is the key, for non-leaf node is the children 218 | 219 | #### B+ tree searching 220 | 221 | Searching from the root, and traverse the node, until find a bigger element, then go into this child 222 | 223 | Find until leaf, run `binary_search` and return the `key` found 224 | 225 | #### B+ tree insertion 226 | 227 | First find the position to insert 228 | 229 | Then there are two types of insertion: 230 | 231 | 1. If the node is not full, then we can directly insert into the leaf node 232 | 2. If the node is full, we need to split. Do the following procedure recursively: 233 | 234 | - Split at the middle of the current node 235 | - Apply a new node as sibling 236 | - Move the second half to the sibling 237 | - If a new `fa` node is needed, apply a new node 238 | - Rearrange the pointers' relationship 239 | 240 | #### B+ tree deletion 241 | 242 | First find the position to delete 243 | 244 | Then there are steps to adjust the tree: 245 | 246 | 1. We need to replace the internal marks with the node's successor 247 | 248 | 2. If the node's size is more than a half or it is root then we can delete directly. 249 | On other cases, we need toIf the node's size is less than a half, we should try to rotate with a neighbour, either the previous one or the next one 250 | 251 | 3. If no neighbour is able to rotate here, we need to merge the node with the neighbour 252 | 253 | 4. If the only key is deleted from the root, we need to decrease the height 254 | 255 | 256 | 257 | All the above operations' detail is illustrated in the comment among the codes. 258 | 259 | Since it is too long and takes too much unnecessary space, here I just give the basic idea instead of implement details. -------------------------------------------------------------------------------- /db/README.md: -------------------------------------------------------------------------------- 1 | # MiniSQL Region 2 | 3 | **Disclaimer: This project is the final project of a course, which can only be considered as a toy. It shows some undergraduates' ideas on how to build a proper database. Suggestions are welcomed.** 4 | 5 | This is `MiniSQL Region`, the final project of *Database System Concept* course. 6 | 7 | Currently, the **HTTP server version** of this project, which is called *Region Server*, is a component of `MiniSQL cluster`, a distributed database cluster. This project also has a **standalone CLI version**, which will be discussed later in this document. 8 | 9 | ![MiniSQL.png](https://i.loli.net/2021/07/11/quRsbFZpzL5r7gY.png) 10 | 11 | > This screen shot shows MiniSQL standalone version 12 | 13 | In this project, we are required to implement a *toy* database engine, which contains some basic features, like creating, dropping and selecting table, inserting and deleting rows, creating and dropping indexes which is implemented by a B+ tree, and providing persistent storage. There are three data types that must be supported: 14 | `INT`, `FLOAT` and `CHAR()`. `UNIQUE` and `PRIMARY KEY` needs to be correctly declared in the DDL, and processed properly. The querying condition could be concatenated using `AND`, and common operators in the condition expression must be supported. External `.sql` files are required to be able to execute by `EXECFILE` statement. 15 | 16 | ## Infrastructure 17 | 18 | ![MiniSQL.png](https://raw.githubusercontent.com/RalXYZ/repo-pictures/47c8cad9a82fa1a3ce59af6799a0c03ddffd4954/MiniSQL/minisql.svg) 19 | 20 | The project contains six modules: *Interpreter*, *API*, *Catalog Manager*, *Record Manager*, *Buffer Manager*, and *Index Manager*. 21 | A golang HTTP server is provided to serve the database, while you can also choose to use the CLI version of this database. 22 | 23 | ## Build 24 | 25 | Firstly, make sure you've installed *CMake*, *Bison* and *Flex*. 26 | Golang should also be installed, which version must be higher than 1.17. 27 | Then, in the project director, run the following command: 28 | 29 | ```shell 30 | cmake . 31 | make 32 | ``` 33 | 34 | Then, two executables, one called `MiniSQL` and another called `minisql_local` will appear in the project directory. 35 | - `minisql_region` is expected to be executed on the server, which exposes HTTP API. Since it is a component of MiniSQL cluster, it is started by master node. 36 | - `minisql_local` is the standalone version, which has a CLI interface. 37 | 38 | If you haven't installed *GNU Readline* library, the program will fallback, using `std::cin` to read input lines, 39 | which will affect the using experience. 40 | 41 | ## Usage 42 | 43 | Start MiniSQL local by the following command: 44 | ```sh 45 | ./minisql_local 46 | ``` 47 | 48 | After the executable has been executed, press `TAB`, then you will see all valid keywords. 49 | Pressing `TAB` can also auto-complete the keywords. 50 | 51 | These are some examples of the supported SQL statements: 52 | ```sql 53 | CREATE TABLE foo (id INT, bar FLOAT, buzz CHAR(255), PRIMARY KEY id); -- valid length of CHAR is between 1 and 255 54 | CREATE TABLE foo (id INT); -- a table can have no primary key 55 | CREATE INDEX bar_idx ON foo (bar); 56 | SELECT * FROM foo; -- brute-force select 57 | SELECT bar, id FROM foo WHERE buzz <> 'bee' AND bar > 3.14; 58 | INSERT INTO foo VALUES (4, 2.71, 'bomb'); 59 | DELETE FROM foo; -- delete all 60 | DELETE FROM foo WHERE id = 5; 61 | DROP INDEX bar_idx; 62 | DROP TABLE foo; 63 | EXECFILE your_sql_script.sql; 64 | QUIT; 65 | ``` 66 | -------------------------------------------------------------------------------- /db/RecordManager/Readme.md: -------------------------------------------------------------------------------- 1 | ## Record Manager module design report 2 | 3 | Writer: 3190104509 Gao Weiyuan 4 | 5 | ### Overview 6 | 7 | `RecordManager ` manages records in database. It supports three major operations about records: insertion, selection and deletion. Also, it offer interfaces to `API` and `IndexManager`. 8 | 9 | ### Module Design 10 | 11 | `RecordManager` is responsible for organizing data in data block returned by `BufferManager`. We store each table in a single file. Because every records in one table has the fixed length and records don't need to across blocks, we can store records in a simple way. We write the records into block linerly, and for ease of deleting, I add a valid sign at the head of the record. The valid sign being zero represents this record is deleted. 12 | 13 | When traversing a table, we firstly the maxmun blockID through `BufferManager`, then we request block one by one. For each block, we traverse the records in it. Because the length of the record is fixed, we easily find the head of each record and then get the information. 14 | 15 | ### implementation 16 | 17 | - Data Structure 18 | 19 | - struct table is defined as follow: 20 | 21 | ```c++ 22 | struct table { 23 | table() {}; 24 | table(std::string table_name, int attr_cnt) 25 | :name(table_name), attribute_cnt(attr_cnt){}; 26 | /* table(const table &T) : name(T.name), attribute_cnt(T.attribute_cnt), record_len(T.record_len), 27 | record_cnt(T.record_cnt), size(T.size), attribute_type(T.attribute_type), attribute_names(T.attribute_names), 28 | indexNames(T.indexNames) {};*/ 29 | 30 | std::string name; 31 | int attribute_cnt, record_len, record_cnt, size; 32 | 33 | std::vector attribute_type; 34 | std::vector attribute_names; 35 | // for index, first stands for attr name, second stands for index name. 36 | std::vector> index; 37 | 38 | friend std::ostream &operator<<(std::ostream &os, const table &tab) { 39 | os << "name: " << tab.name << " attribute_cnt: " << tab.attribute_cnt << " record_len: " << tab.record_len 40 | << " record_cnt: " << tab.record_cnt << " size: " << tab.size 41 | << " attribute_names: " << tab.attribute_names.size(); 42 | return os; 43 | } 44 | }; 45 | ``` 46 | 47 | - sql_value 48 | each item in recors is a sql_value 49 | 50 | ```c++ 51 | struct sql_value { 52 | sql_value_type sql_type; 53 | int sql_int; 54 | float sql_float; 55 | std::string sql_str; 56 | 57 | sql_value() = default; 58 | sql_value(int i) : sql_int(i), sql_type(value_type::INT) {}; 59 | sql_value(float f) : sql_float(f), sql_type(value_type::FLOAT) {}; 60 | sql_value(std::string& str) : sql_str(str), sql_type(value_type::CHAR) {}; 61 | 62 | void reset() { 63 | sql_int = 0; 64 | sql_float = 0; 65 | sql_str.clear(); 66 | } 67 | 68 | std::string toStr() const { 69 | switch (sql_type.type) { 70 | case value_type::INT: 71 | return std::to_string(sql_int); 72 | case value_type::FLOAT: 73 | return std::to_string(sql_float); 74 | case value_type::CHAR: 75 | return this->sql_str; 76 | } 77 | } 78 | 79 | bool operator<(const sql_value &e) const { 80 | switch (sql_type.type) { 81 | case value_type::INT: 82 | return sql_int < e.sql_int; 83 | case value_type::FLOAT: 84 | return sql_float < e.sql_float; 85 | case value_type::CHAR: 86 | return sql_str < e.sql_str; 87 | default: 88 | throw std::runtime_error("Undefined Type!"); 89 | } 90 | } 91 | 92 | bool operator==(const sql_value &e) const { 93 | switch (sql_type.type) { 94 | case value_type::INT: 95 | return sql_int == e.sql_int; 96 | case value_type::FLOAT: 97 | return sql_float == e.sql_float; 98 | case value_type::CHAR: 99 | return sql_str == e.sql_str; 100 | default: 101 | throw std::runtime_error("Undefined Type!"); 102 | } 103 | } 104 | 105 | bool operator!=(const sql_value &e) const { return !operator==(e); } 106 | 107 | bool operator>(const sql_value &e) const { return !operator<(e) && operator!=(e); } 108 | 109 | bool operator<=(const sql_value &e) const { return operator<(e) || operator==(e); } 110 | 111 | bool operator>=(const sql_value &e) const { return !operator<(e); } 112 | }; 113 | ``` 114 | 115 | - sql_value_type 116 | 117 | the type of the value is a struct sql_value_type: 118 | 119 | ```c++ 120 | struct sql_value_type { 121 | value_type type; 122 | uint8_t length = 0; 123 | bool primary = false, unique = false; 124 | uint8_t size() const{ 125 | switch(type){ 126 | case value_type::INT: 127 | return sizeof(int); 128 | case value_type::FLOAT: 129 | return sizeof(float); 130 | case value_type::CHAR: 131 | return length + 1; 132 | } 133 | } 134 | 135 | sql_value_type() = default; 136 | sql_value_type(value_type type) : type(type) {} 137 | sql_value_type(uint8_t length) : type(value_type::CHAR), length(length) {} 138 | }; 139 | ``` 140 | 141 | - sql_tuple 142 | A sql_tuple is used to represent a record, it is a vector of sql_value essentially. 143 | 144 | ```c++ 145 | struct sql_tuple { 146 | std::vector element; 147 | 148 | /* 149 | * attrTable: all columns 150 | * attrFetch: columns to be selected 151 | */ 152 | row fetchRow(const std::vector &attrTable, const std::vector &attrFetch) const { 153 | row row; 154 | bool attrFound; 155 | row.col.reserve(attrFetch.size()); 156 | for (auto fetch : attrFetch) { 157 | attrFound = false; 158 | for (int i = 0; i < attrTable.size(); i++) { 159 | if (fetch == attrTable[i]) { 160 | row.col.push_back(element[i].toStr()); 161 | attrFound = true; 162 | break; 163 | } 164 | } 165 | if (!attrFound) { 166 | std::cerr << "Undefined attr in row fetching!!" << std::endl; 167 | } 168 | } 169 | return row; 170 | } 171 | }; 172 | ``` 173 | 174 | - Condition 175 | To represent conditons of select and deletion statement, we also define condition struct. 176 | 177 | ```c++ 178 | struct condition { 179 | std::string attribute_name; 180 | attribute_operator op; 181 | sql_value value; 182 | 183 | condition() = default; 184 | condition(std::string& attribute_name, attribute_operator op, sql_value& value) : 185 | attribute_name(attribute_name), 186 | op(op), 187 | value(value) {} 188 | 189 | bool test(const sql_value &e) const { 190 | switch (op) { 191 | case attribute_operator::EQUAL: 192 | return e == value; 193 | case attribute_operator::NOT_EQUAL: 194 | return e != value; 195 | case attribute_operator::GREATER: 196 | return e > value; 197 | case attribute_operator::GREATER_EQUAL: 198 | return e >= value; 199 | case attribute_operator::LESS: 200 | return e < value; 201 | case attribute_operator::LESS_EQUAL: 202 | return e <= value; 203 | default: 204 | std::cerr << "Undefined condition width cond !" << std::endl; 205 | } 206 | } 207 | }; 208 | ``` 209 | 210 | - Major Function 211 | 212 | - Creat Table 213 | 214 | ```c++ 215 | bool creatTable(const string &tableName); 216 | ``` 217 | 218 | tableName: the name of table. 219 | 220 | function: Creat data file via `BufferManager`. ``BufferManager`` will create a file which type is ``.tb`` 221 | 222 | - Drop Table 223 | 224 | ```c++ 225 | bool dropTable(const string &tableName); 226 | ``` 227 | 228 | tableName: the name of table. 229 | 230 | function: Remove data file via `BufferManager`, ``BufferManager`` firstly call closeFile() to close the file, then remove the file in disk. 231 | 232 | - Insert Record 233 | 234 | ```c++ 235 | int insertRecord(const macro::table &table, const sql_tuple &record); 236 | ``` 237 | 238 | table: the table instance 239 | 240 | record: the record to be insert 241 | 242 | return: the insert position in the table 243 | 244 | function: Insert a record. Records are stored in data file block by block. Records in the same table will be convert to a fixed-length and won't be stored across the block. This function traverse the records one by one until it find a record whose valid sign is zero. Then put the record into that slot and return the position. 245 | 246 | - Delete Record 247 | 248 | ```c++ 249 | vector> deleteRecord(const macro::table &table, const vector &conditions); 250 | ``` 251 | 252 | table: the table instance 253 | 254 | conditions: the delete conditions 255 | 256 | return: a vector, each element in vector is a pair which contains a deleted record and its position. 257 | 258 | function: Delete records with a batch of conditions. When doing deletion, we traverse the whole table and delete the records which satisfy the conditions. This opration can be optimized in `IndexManager` using B+ Tree. 259 | 260 | - Select Record 261 | 262 | ```c++ 263 | result selectRecord(const macro::table &table, const vector &attr, const vector &cond); 264 | ``` 265 | 266 | table: the table instance 267 | 268 | attr: the columns to be selected, because our mini sql is not support for partial selection, this parameter is always equal to all columns of the table. 269 | 270 | conditions: the select conditions 271 | 272 | return: a vector, each element in vector is a pair which contains a selected record and its position. 273 | 274 | Select record with a batch of conditions. Similar to deletion, we traverse the whole table and select the records which satisfy the conditions. This opration can be optimized in `IndexManager` using B+ Tree. Not support for selecting part columns. 275 | 276 | - Get Record 277 | 278 | ```c++ 279 | sql_tuple getRecord(const macro::table &table, uint32_t id); 280 | ``` 281 | 282 | table: the table instance 283 | 284 | id: the position 285 | 286 | return: the record in that position. 287 | 288 | function: Get a record by its positon in data file, mainly used by index. 289 | 290 | - Get Record Pairs 291 | 292 | ```c++ 293 | vector> getRecordPairs(const macro::table &table); 294 | ``` 295 | 296 | table: the table instance 297 | 298 | return: a vector, each element in vector is a pair which contains a record and its position. 299 | 300 | function: Return all records paired with their position in data file, mainly used by index. -------------------------------------------------------------------------------- /db/RecordManager/RecordManager.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Apple on 2021/6/12. 3 | // 4 | 5 | #include"RecordManager.h" 6 | #include "../utils/exception.h" 7 | 8 | using namespace std; 9 | 10 | extern BufferManager buf_mgt; 11 | 12 | bool RecordManager::creatTable(const string &tableName) { 13 | string tableFileStr = macro::tableFile(tableName); 14 | BufferManager::createFile(tableFileStr); 15 | return true; 16 | } 17 | 18 | bool RecordManager::dropTable(const string &tableName) { 19 | string tableFileStr = macro::tableFile(tableName); 20 | buf_mgt.removeFile(tableFileStr); 21 | return true; 22 | } 23 | 24 | int RecordManager::insertRecord(const macro::table &table, const sql_tuple &record) { 25 | string tableFileStr = macro::tableFile(table.name); 26 | int blockID = buf_mgt.getBlockCnt(tableFileStr) - 1; 27 | Block *block = &buf_mgt.getBlock(tableFileStr, blockID); 28 | 29 | char *content = block->blockContent; 30 | 31 | int recordLen = table.record_len + 1; 32 | int recordsPreBlock = macro::BlockSize / recordLen; 33 | int recordOffset = 0; 34 | bool isValid = false; 35 | while (recordOffset < recordsPreBlock) { 36 | if (content[recordOffset * recordLen] != 0) { 37 | recordOffset++; 38 | continue; 39 | } 40 | isValid = true; 41 | content += recordOffset * recordLen; 42 | break; 43 | } 44 | 45 | if (!isValid) { 46 | recordOffset = 0; 47 | buf_mgt.unlock(tableFileStr, blockID); 48 | block = &buf_mgt.getBlock(tableFileStr, ++blockID); 49 | content = block->blockContent; 50 | } 51 | 52 | content[0] = 1; 53 | int offset = 1; 54 | 55 | string str; 56 | int lenOffset; 57 | auto attr_type = table.attribute_type.begin(); 58 | for (auto attr = record.element.begin(); attr < record.element.end(); attr_type++, attr++) { 59 | switch (attr_type->type) { 60 | case value_type::INT: 61 | memcpy(content + offset, &attr->sql_int, sizeof(int)); 62 | offset += sizeof(int); 63 | break; 64 | case value_type::CHAR: 65 | str = attr->sql_str; 66 | lenOffset = attr_type->length + 1 - str.length(); 67 | if (lenOffset > 0) { 68 | str.insert(str.length(), lenOffset, 0); 69 | } 70 | memcpy(content + offset, str.c_str(), attr_type->length + 1); 71 | offset += attr_type->length + 1; 72 | break; 73 | case value_type::FLOAT: 74 | memcpy(content + offset, &attr->sql_float, sizeof(float)); 75 | offset += sizeof(float); 76 | break; 77 | } 78 | } 79 | 80 | buf_mgt.setDirty(tableFileStr, blockID); 81 | return blockID * recordsPreBlock + recordOffset; 82 | } 83 | 84 | vector> RecordManager::deleteRecord(const macro::table &table, const vector &conditions) { 85 | string tableFileStr = macro::tableFile(table.name); 86 | int length = table.record_len + 1; 87 | int recordCnt = macro::BlockSize / length; 88 | int blockCnt = buf_mgt.getBlockCnt(tableFileStr); 89 | vector> deleteRec; 90 | sql_tuple tup; 91 | 92 | for(int blockID = 0; blockID < blockCnt; blockID++) { 93 | Block &block = buf_mgt.getBlock(macro::tableFile(table.name), blockID); 94 | char *content = block.blockContent; 95 | for (int i = 0; i < recordCnt; i++) { 96 | if (content[i * length] == 0) { continue; } 97 | tup = genTuple(content, i * length, table.attribute_type); 98 | if (condsTest(conditions, tup, table.attribute_names)) { 99 | content[i * length] = 0; 100 | deleteRec.emplace_back(make_pair(blockID * recordCnt + i, tup)); 101 | } 102 | } 103 | buf_mgt.setDirty(macro::tableFile(table.name), blockID); 104 | buf_mgt.unlock(macro::tableFile(table.name), blockID); 105 | } 106 | 107 | return deleteRec; 108 | } 109 | 110 | /* 111 | * if attr is empty, this means that a SELECT * has been executed 112 | */ 113 | result RecordManager::selectRecord(const macro::table &table, const vector &attr, const vector &cond) { 114 | string tableFileStr = macro::tableFile(table.name); 115 | 116 | int recordLen = table.record_len + 1; 117 | int recordCnt = macro::BlockSize / recordLen; 118 | int blockCnt = buf_mgt.getBlockCnt(tableFileStr); 119 | 120 | sql_tuple tup; 121 | row row; 122 | result res; 123 | for(int blockID = 0; blockID < blockCnt; blockID++) { 124 | Block &block = buf_mgt.getBlock(tableFileStr, blockID); 125 | char *content = block.blockContent; 126 | for (int i = 0; i < recordCnt; i++) { 127 | if (content[i * recordLen] == 0) { continue; } 128 | tup = genTuple(content, i * recordLen, table.attribute_type); 129 | if (condsTest(cond, tup, table.attribute_names)) { 130 | if (attr.empty()) { 131 | row = tup.fetchRow(table.attribute_names, table.attribute_names); 132 | } else { 133 | row = tup.fetchRow(table.attribute_names, attr); 134 | } 135 | res.row.push_back(row); 136 | } 137 | } 138 | buf_mgt.unlock(macro::tableFile(table.name), blockID); 139 | } 140 | return res; 141 | } 142 | 143 | sql_tuple RecordManager::getRecord(const macro::table &table, uint32_t id) { 144 | string tableFileStr = macro::tableFile(table.name); 145 | 146 | int recordLen = table.record_len + 1; 147 | int recordCnt = macro::BlockSize / recordLen; 148 | int blockCnt = buf_mgt.getBlockCnt(tableFileStr); 149 | int blockId = id / recordCnt; 150 | if(blockId + 1 > blockCnt) { 151 | throw sql_exception(501, "record manager", "record \'" + std::to_string(id) + "\' beyond range"); 152 | } 153 | 154 | Block &block = buf_mgt.getBlock(tableFileStr, blockId); 155 | char *content = block.blockContent; 156 | int recordOffset = id - blockId * recordCnt; 157 | if(content[recordOffset * recordLen] != 1) { 158 | throw sql_exception(502, "record manager", "invalid record"); 159 | } 160 | 161 | sql_tuple tup = genTuple(content, recordOffset * recordLen, table.attribute_type); 162 | return tup; 163 | } 164 | 165 | vector> RecordManager::getRecordPairs(const macro::table &table) { 166 | string tableFileStr = macro::tableFile(table.name); 167 | 168 | int recordLen = table.record_len + 1; 169 | int recordCnt = macro::BlockSize / recordLen; 170 | int blockCnt = buf_mgt.getBlockCnt(tableFileStr); 171 | vector> res; 172 | sql_tuple tup; 173 | 174 | for(int blockID = 0; blockID < blockCnt; blockID++) { 175 | Block &block = buf_mgt.getBlock(tableFileStr, blockID); 176 | char *content = block.blockContent; 177 | for (int i = 0; i < recordCnt; i++) { 178 | if (content[i * recordLen] == 0) { continue; } 179 | tup = genTuple(content, i * recordLen, table.attribute_type); 180 | res.emplace_back(blockID * recordLen + i, tup); 181 | } 182 | buf_mgt.unlock(macro::tableFile(table.name), blockID); 183 | } 184 | 185 | return res; 186 | } 187 | 188 | void RecordManager::printResult(const result &res) { 189 | for (auto const &row : res.row) { 190 | cout << " | "; 191 | for (auto const &col : row.col) { 192 | cout << col << " | "; 193 | } 194 | cout << endl; 195 | } 196 | } 197 | 198 | bool RecordManager::condsTest(const std::vector &conds, const sql_tuple &tup, const std::vector &attr) { 199 | int condPos; 200 | for (const condition& cond : conds) { 201 | condPos = -1; 202 | for (int i = 0; i < attr.size(); i++) { 203 | if (attr[i] == cond.attribute_name) { 204 | condPos = i; 205 | break; 206 | } 207 | } 208 | if (condPos == -1) { 209 | std::cerr << "Attr not found in cond test!" << std::endl; 210 | } 211 | if (!cond.test(tup.element[condPos])) { 212 | return false; 213 | } 214 | } 215 | return true; 216 | } 217 | 218 | sql_tuple RecordManager::genTuple(const char *content, int offset, const std::vector &attrType) { 219 | const char *curContent = content + offset + 1; // 1 for meta bit 220 | sql_value e; 221 | sql_tuple tup; 222 | tup.element.clear(); 223 | for (auto i : attrType) { 224 | e.reset(); 225 | e.sql_type = i; 226 | switch (i.type) { 227 | case value_type::INT: 228 | memcpy(&e.sql_int, curContent, sizeof(int)); 229 | curContent += sizeof(int); 230 | break; 231 | case value_type::FLOAT: 232 | memcpy(&e.sql_float, curContent, sizeof(float)); 233 | curContent += sizeof(float); 234 | break; 235 | case value_type::CHAR: 236 | e.sql_str = curContent; 237 | curContent += i.length + 1; 238 | break; 239 | } 240 | tup.element.push_back(e); 241 | } 242 | return tup; 243 | } 244 | -------------------------------------------------------------------------------- /db/RecordManager/RecordManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Apple on 2021/6/12. 3 | // 4 | 5 | #ifndef MINISQL_RECORDMANAGER_H 6 | #define MINISQL_RECORDMANAGER_H 7 | 8 | #include 9 | #include "../BufferManager/BufferManager.h" 10 | #include "../macro.h" 11 | 12 | using namespace std; 13 | 14 | class RecordManager { 15 | public: 16 | RecordManager() = default; 17 | ~RecordManager() = default; 18 | 19 | sql_tuple getRecord(const macro::table &table, uint32_t id); 20 | 21 | vector> getRecordPairs(const macro::table &table); 22 | 23 | bool creatTable(const string &tableName); 24 | 25 | bool dropTable(const string &tableName); 26 | 27 | bool createIndex(const macro::table &table, const sql_value_type &index); 28 | 29 | bool dropIndex(const macro::table &table, const string &index); 30 | 31 | [[nodiscard]] 32 | int insertRecord(const macro::table &table, const sql_tuple &record); 33 | 34 | vector> deleteRecord(const macro::table &table, const vector &conditions); 35 | 36 | [[nodiscard]] 37 | result selectRecord(const macro::table &table, const vector &attr, const vector &cond); 38 | 39 | static void printResult(const result &res) ; 40 | 41 | private: 42 | 43 | static sql_tuple genTuple(const char *blockBuffer, int offset, const std::vector &attrType); 44 | 45 | static bool condsTest(const vector &conds, const sql_tuple &tup, const vector &attr); 46 | }; 47 | 48 | 49 | #endif //MINISQL_RECORDMANAGER_H -------------------------------------------------------------------------------- /db/api/README.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | *API* is the separator of the front-end part and the back-end part of these project. The front-end part only contains the *Interpreter*, which eventually creates aN AST OBJECT. *API* handles these AST objects by calling the corresponding functions. 4 | *API* contains a base class. The name of the base class is literally `api::base`, which is almost a virtual class. Every SQL statement extends from this base class, and implements the `exec` function of the base class. This function is responsible for calling other modules in the back-end. For an example, `api::create_table::exec` needs to call *Catalog Manger* to and the meta-data of the table into the schema, then calls *Record Manager* to create an record file of this table, then may call *Index Manager* to create a B+ tree for the index on primary key. The only back-end module that will not be called directly by *API* is *Buffer Manager*, which is managed by *Record Manager*. 5 | *API* also manages the query result, the query duration, the exception, and needs to print the result of `SELECT` statement into a table properly. Everything related to direct interaction with the user are not related to any module below *API*. 6 | 7 | 8 | ## API base class and it's derived classes 9 | 10 | The whole API has two types of classes: a base class named `api::base`, and multiple derived classes extended from the base class. The base class is declared as following: 11 | 12 | ```c++ 13 | struct base { 14 | virtual ~base() = default; 15 | virtual void exec() {}; 16 | }; 17 | ``` 18 | 19 | There are only two functions defined in base class, both of them are all virtual functions. To explicitly set the default destructor as a virtual function is because of the language feature of C++, and I do not want to explain it here. What is important is the second function `exec()`, which is a virtual function, and will be overridden by the derived classes. Here is an example of the derived class: 20 | 21 | ```c++ 22 | class create_table final : public base { 23 | std::string table_name; 24 | std::vector schema_list; 25 | std::string primary_key; 26 | 27 | void exec() override; 28 | 29 | public: 30 | create_table(std::string& table_name, 31 | std::vector& schema_list, 32 | std::string& primary_key) : 33 | table_name(table_name), schema_list(schema_list), primary_key(primary_key) {} 34 | }; 35 | ``` 36 | 37 | This class will be instantiated after an SQL CREATE TABLE instruction has been correctly parsed. To instantiate, the constructor will be called, giving each private variable a specific value. The `exec()` function has been defined in another source code file, overriding the same virtual function in the base class. Virtual function is used to make the object passing more simple. The only things we need to write in the parser source code are: 38 | 39 | ```c++ 40 | extern std::unique_ptr query_object_ptr; 41 | // in some of the leaf nodes of the parser tree: 42 | query_object_ptr = std::make_unique($3, $5, $6); 43 | // in other leaf nodes: 44 | query_object_ptr = std::make_unique($3); 45 | ``` 46 | 47 | Since both `api::create_table` and `api::execfile` are extended from `api::base`, calling `query_object_ptr->exec()` will call the corresponding `exec()` function of the derived class object pointed by `query_object_ptr`, while `query_object_ptr` is a unique pointer of `api::base`. 48 | 49 | ## Inside `exec()` function 50 | 51 | The procedure inside `exec()` is what the API perform. Usually, it calls record manager, catalog manager and buffer manager, to perform the corresponding features. The implementation is highly case sensitive, so that I will not introduce them one by one here. 52 | 53 | ## Output of SELECT 54 | 55 | The output of SELECT is also handled by API. Here is an example of the output: 56 | 57 | ``` 58 | MiniSQL> SELECT * FROM foo; 59 | +-----+-----------+------+-----------+ 60 | | id | bar | buzz | barbar | 61 | +-----+-----------+------+-----------+ 62 | | 1 | 51.279999 | 4 | 9.137000 | 63 | | 2 | 45.189999 | 2 | 78.040001 | 64 | | 3 | 18.049999 | 9 | 33.220001 | 65 | | 4 | 98.809998 | 8 | 41.119999 | 66 | | 5 | 19.690001 | 9 | 43.980000 | 67 | | 6 | 35.400002 | 10 | 74.980003 | 68 | | 7 | 46.139999 | 10 | 17.040001 | 69 | +-----+-----------+------+-----------+ 70 | ``` 71 | 72 | To implement this correctly, we need to calculate the max length of each column. This can be done by iterating the whole table. Then, use `std::setw()` and `std::setfill()` to help us to construct the format of output. 73 | 74 | ## Error handling, result returning and time keeping 75 | 76 | We have implemented a custom exception type, which is extended from `std::exception`: 77 | 78 | ```c++ 79 | class sql_exception : public std::exception { 80 | unsigned int code; 81 | std::string module_name; 82 | std::string detail; 83 | 84 | public: 85 | sql_exception() = default; 86 | sql_exception(unsigned int code, std::string& module_name, std::string& detail) : 87 | code(code), module_name(module_name), detail(detail) {} 88 | 89 | sql_exception(unsigned int code, std::string&& module_name, std::string&& detail) : 90 | code(code), module_name(module_name), detail(detail) {} 91 | 92 | friend std::ostream& operator<<(std::ostream& os, const sql_exception& e); 93 | }; 94 | 95 | std::ostream& operator<<(std::ostream& os, const sql_exception& e) { 96 | os << "ERROR " << e.code << " from <" << e.module_name << ">: " << e.detail; 97 | return os; 98 | } 99 | ``` 100 | 101 | When something goes wrong in the project, an exception will be thrown. The exception must be in `sql_exception` type, otherwise it cannot be handled correctly. After the exception has been handled, `std::cout` will be called to print the exception. 102 | 103 | The result printer and time keeper also prints data at the end of execution. 104 | -------------------------------------------------------------------------------- /db/api/api.cpp: -------------------------------------------------------------------------------- 1 | #include "api.h" 2 | #include "utils.h" 3 | #include 4 | #include 5 | 6 | #include "../RecordManager/RecordManager.h" 7 | #include "../CatalogManager/CatalogManager.h" 8 | #include "../IndexManager/IndexManager.h" 9 | #include "../utils/exception.h" 10 | #include "../utils//utils.h" 11 | 12 | extern RecordManager rec_mgt; 13 | extern CatalogManager cat_mgt; 14 | extern BufferManager buf_mgt; 15 | extern IndexManager idx_mgt; 16 | extern std::string file_to_exec; 17 | 18 | namespace api { 19 | void base::exec() { 20 | throw sql_exception(209, "api", "no query specified"); 21 | } 22 | 23 | void create_table::exec() { 24 | throw_on_table_exist(this->table_name); 25 | 26 | std::unordered_set attribute_duplication_check; 27 | 28 | for (const auto &i : this->schema_list) { 29 | if (i.type.type == value_type::CHAR && i.type.length == 0) { 30 | throw sql_exception(202, "api", 31 | "char \'" + i.name + "\' needs to have a length between 1 and 255"); 32 | } 33 | if (attribute_duplication_check.contains(i.name)) { 34 | throw sql_exception(210, "api", "duplicate column name \'" + i.name + "\'"); 35 | } else { 36 | attribute_duplication_check.insert(i.name); 37 | } 38 | } 39 | 40 | // create index for primary key 41 | if (!this->primary_key.empty()) { 42 | bool pk_is_valid = false; 43 | value_type pk_type; 44 | for (const auto &i : this->schema_list) { 45 | if (i.name == this->primary_key) { 46 | pk_is_valid = true; 47 | pk_type = i.type.type; 48 | break; 49 | } 50 | } 51 | 52 | if (!pk_is_valid) { 53 | throw sql_exception(206, "api", "invalid primary key \'" + primary_key + "\'"); 54 | } 55 | #ifndef DETACH_INDEX_MANAGER 56 | idx_mgt.create(generate_table_pk_name(this->table_name)); 57 | #endif 58 | } 59 | 60 | rec_mgt.creatTable(this->table_name); 61 | cat_mgt.CreateTable(this->table_name, this->schema_list, this->primary_key); 62 | 63 | std::cout << "query OK, 0 row affected"; 64 | } 65 | 66 | void insert_table::exec() { 67 | throw_on_table_not_exist(this->table_name); 68 | auto& table = cat_mgt.GetTable(this->table_name); 69 | 70 | // validate input amount 71 | if (table.attribute_names.size() != this->insert_list.size()) { 72 | throw sql_exception(203, "api", 73 | "insert attribute amount mismatch: needs " 74 | + std::to_string(table.attribute_names.size()) 75 | + " attributes, got " 76 | + std::to_string(this->insert_list.size())); 77 | } 78 | 79 | // validate the input type 80 | { 81 | for (int i = 0; i < this->insert_list.size(); i++) { 82 | if (table.attribute_type.at(i).type != this->insert_list.at(i).sql_type.type) { 83 | throw sql_exception(208, "api", 84 | "insert attribute type mismatch: at attribute " 85 | + std::to_string(i) 86 | + ", needs " 87 | + return_value_name(table.attribute_type.at(i).type) 88 | + ", got " 89 | + return_value_name(this->insert_list.at(i).sql_type.type)); 90 | } 91 | } 92 | } 93 | 94 | // check unique and duplication 95 | { 96 | std::vector duplication_check_list; 97 | for (int i = 0; i < table.attribute_names.size(); i++) { 98 | if (table.attribute_type.at(i).unique) { 99 | duplication_check_list.emplace_back(table.attribute_names.at(i), 100 | attribute_operator::EQUAL, 101 | this->insert_list.at(i)); 102 | } 103 | } 104 | for (const auto &i: duplication_check_list) { 105 | std::vector cond; 106 | 107 | cond.push_back(i); 108 | auto res = rec_mgt.selectRecord(table, vector(), cond); 109 | if(!res.row.empty()) { 110 | throw sql_exception(207, "api", 111 | "there exists a duplicated value on attribute \'" 112 | + i.attribute_name +"\'"); 113 | } 114 | } 115 | } 116 | 117 | #ifndef DETACH_INDEX_MANAGER // validate 118 | for (const auto &i : table.index) { 119 | // i : pair 120 | int j = 0; 121 | for (; j < table.attribute_names.size(); j++) { 122 | if (i.first == table.attribute_names.at(j)) { 123 | break; 124 | } 125 | } 126 | if (j == table.attribute_names.size()) { 127 | throw sql_exception(211, "api", 128 | "attribute \'" 129 | + i.first 130 | + "\' of index \'" 131 | + i.second 132 | + "\' cannot be found in table \'" 133 | + table_name 134 | + "\'"); 135 | } 136 | 137 | auto found_element = false; 138 | try { 139 | idx_mgt.search(i.second, this->insert_list.at(j), j, table); 140 | found_element = true; 141 | } catch (sql_exception &e) { 142 | if (e.code != 300) { 143 | throw e; 144 | } 145 | } 146 | if (found_element) { 147 | throw sql_exception(213, "api", "duplicated index value: " + this->insert_list.at(j).toStr()); 148 | } 149 | 150 | } 151 | #endif 152 | sql_tuple tuple; 153 | tuple.element = this->insert_list; 154 | const auto index_row_in_record = rec_mgt.insertRecord(table, tuple); 155 | 156 | #ifndef DETACH_INDEX_MANAGER 157 | for (const auto &i : table.index) { 158 | // i : pair 159 | int j = 0; 160 | for (; j < table.attribute_names.size(); j++) { 161 | if (i.first == table.attribute_names.at(j)) { 162 | break; 163 | } 164 | } 165 | auto value = rec_mgt.getRecord(table, index_row_in_record).element.at(j); 166 | idx_mgt.insert(i.second, value, j, table, index_row_in_record); 167 | } 168 | #endif 169 | 170 | table.record_cnt++; 171 | std::cout << "query OK, 1 row affected"; 172 | } 173 | 174 | void select_table::exec() { 175 | throw_on_table_not_exist(this->table_name); 176 | 177 | auto& table = cat_mgt.GetTable(this->table_name); 178 | if (this->attribute_list.empty()) { 179 | this->attribute_list = table.attribute_names; 180 | } 181 | 182 | // validate attribute list, if attribute list is empty, this means SELECT * 183 | { 184 | auto attribute_set = std::unordered_set(); 185 | for (const auto &i : table.attribute_names) { 186 | attribute_set.insert(i); 187 | } 188 | for (const auto &i : this->attribute_list) { 189 | if (attribute_set.contains(i)) { 190 | attribute_set.erase(i); 191 | } else { 192 | throw sql_exception(212, "api", "attribute \'" + i + "\' invalid"); 193 | } 194 | } 195 | } 196 | 197 | validate_condition(table, this->condition_list); 198 | 199 | result res; 200 | 201 | if (this->condition_list.size() == 1 && 202 | this->condition_list.at(0).op == attribute_operator::EQUAL && 203 | !table.index.empty()) { 204 | uint32_t idx_pos = -1; 205 | string treename; 206 | for(auto i = 0; i < table.index.size(); ++i) { 207 | if(this->condition_list.at(0).attribute_name == table.index.at(i).first) { 208 | idx_pos = i; 209 | treename = table.index.at(i).second; 210 | break; 211 | } 212 | } 213 | if (idx_pos != -1) { 214 | auto search_res = idx_mgt.search(treename, this->condition_list.at(0).value, idx_pos, table); 215 | auto record = rec_mgt.getRecord(table, search_res); 216 | res = convert_sql_tuple_to_result(table.attribute_names, this->attribute_list, record); 217 | } 218 | } 219 | if (res.row.empty()) { 220 | res = rec_mgt.selectRecord(table, this->attribute_list, this->condition_list); 221 | } 222 | 223 | print_select_result(this->attribute_list, res); 224 | 225 | if (res.row.empty()) { 226 | std::cout << "empty set"; 227 | } else { 228 | std::cout << res.row.size() << " row(s) in set"; 229 | } 230 | 231 | } 232 | 233 | void delete_table::exec() { 234 | throw_on_table_not_exist(this->table_name); 235 | auto& table = cat_mgt.GetTable(this->table_name); 236 | 237 | validate_condition(table, this->condition_list); 238 | 239 | auto delete_row_count = rec_mgt.deleteRecord(table, this->condition_list); 240 | if (delete_row_count.empty()) { 241 | throw sql_exception(205, "api", "delete failed"); 242 | } 243 | 244 | #ifndef DETACH_INDEX_MANAGER 245 | unordered_map umap; 246 | for (const auto &x: delete_row_count) 247 | umap.insert(x); 248 | 249 | for (const auto &i : table.index) { 250 | // i : pair 251 | int j = 0; 252 | for (; j < table.attribute_names.size(); j++) { 253 | if (i.first == table.attribute_names.at(j)) { 254 | break; 255 | } 256 | } 257 | if (j == table.attribute_names.size()) { 258 | throw sql_exception(211, "api", 259 | "attribute \'" 260 | + i.first 261 | + "\' of index \'" 262 | + i.second 263 | + "\' cannot be found in table \'" 264 | + table_name 265 | + "\'"); 266 | } 267 | // TODO: remove all deleted rows in B+ tree 268 | for(auto &x: delete_row_count) 269 | idx_mgt.remove(i.second, x.second.element.at(j), j, table, umap); 270 | } 271 | #endif 272 | 273 | std::cout << "query OK, " << delete_row_count.size() << " row(s) affected"; 274 | } 275 | 276 | void drop_table::exec() { 277 | throw_on_table_not_exist(this->table_name); 278 | auto table = cat_mgt.GetTable(this->table_name); 279 | 280 | #ifndef DETACH_INDEX_MANAGER 281 | // drop every index in the table 282 | for (auto & i : table.index) { 283 | idx_mgt.drop(i.second); 284 | } 285 | #endif 286 | 287 | cat_mgt.DropTableByName(this->table_name); 288 | cat_mgt.Flush(); 289 | 290 | rec_mgt.dropTable(this->table_name); 291 | 292 | std::cout << "table \'" << this->table_name << "\' has been dropped"; 293 | } 294 | 295 | void create_index::exec() { 296 | throw_on_table_not_exist(this->table_name); 297 | auto& table = cat_mgt.GetTable(this->table_name); 298 | 299 | for (const auto &i : table.attribute_names) { 300 | 301 | } 302 | 303 | // i : pair 304 | int column = 0; 305 | for (; column < table.attribute_names.size(); column++) { 306 | if (this->attribute_name == table.attribute_names.at(column)) { 307 | break; 308 | } 309 | } 310 | if (column == table.attribute_names.size()) { 311 | throw sql_exception(212, "api", 312 | "attribute \'" 313 | + this->attribute_name 314 | + "\' cannot be found in table \'" 315 | + table_name 316 | + "\'"); 317 | } 318 | 319 | cat_mgt.CreateIndex(table, this->attribute_name, this->index_name); 320 | 321 | #ifndef DETACH_INDEX_MANAGER 322 | auto record_row_tuple_pairs = rec_mgt.getRecordPairs(table); 323 | 324 | auto record_to_insert = std::vector>(); 325 | for (const auto &iter : record_row_tuple_pairs) { 326 | record_to_insert.emplace_back(iter.first, iter.second.element.at(column)); 327 | } 328 | idx_mgt.create(this->index_name, record_to_insert, column, table); 329 | #endif 330 | std::cout << "index \'" << this->index_name << "\' created on \'" << this->table_name << "." << this->attribute_name << "\'"; 331 | } 332 | 333 | void drop_index::exec() { 334 | cat_mgt.DropIndex(this->index_name); 335 | #ifndef DETACH_INDEX_MANAGER 336 | idx_mgt.drop(this->index_name); 337 | #endif 338 | std::cout << "index \'" << this->index_name << "\' dropped"; 339 | } 340 | 341 | void execfile::exec() { 342 | file_to_exec = this->filename; 343 | std::cout << "context switched to file \'" << filename << "\'"; 344 | } 345 | 346 | void exit::exec() { 347 | buf_mgt.closeAllFile(); 348 | cat_mgt.Flush(); 349 | idx_mgt.save(); 350 | std::cout << "bye!" << std::endl; 351 | std::exit(0); 352 | } 353 | } 354 | 355 | -------------------------------------------------------------------------------- /db/api/api.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_API_H 2 | #define MINISQL_API_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "../macro.h" 9 | 10 | namespace api { 11 | struct base { 12 | /* 13 | * If expression evaluates to a pointer to a base class subobject 14 | * of the object that was allocated with new, the destructor of 15 | * the base class must be virtual, otherwise the behavior is undefined. 16 | */ 17 | virtual ~base() = default; 18 | 19 | virtual void exec(); 20 | }; 21 | 22 | class create_table final : public base { 23 | std::string table_name; 24 | std::vector schema_list; 25 | std::string primary_key; 26 | 27 | void exec() override; 28 | 29 | public: 30 | create_table(std::string& table_name, 31 | std::vector& schema_list, 32 | std::string& primary_key) : 33 | table_name(table_name), schema_list(schema_list), primary_key(primary_key) {} 34 | }; 35 | 36 | class create_index final : public base { 37 | std::string index_name; 38 | std::string table_name; 39 | std::string attribute_name; 40 | 41 | void exec() override; 42 | 43 | public: 44 | create_index(std::string& index_name, 45 | std::string& table_name, 46 | std::string& attribute_name) : 47 | index_name(index_name), table_name(table_name), attribute_name(attribute_name) {} 48 | }; 49 | 50 | class delete_table final : public base { 51 | std::string table_name; 52 | std::vector condition_list; 53 | 54 | void exec() override; 55 | 56 | public: 57 | explicit delete_table(std::string& table_name, 58 | std::vector& condition_list) : 59 | table_name(table_name), condition_list(condition_list) {} 60 | }; 61 | 62 | class drop_table final : public base { 63 | std::string table_name; 64 | 65 | void exec() override; 66 | 67 | public: 68 | explicit drop_table(std::string &table_name) : table_name(table_name) {} 69 | }; 70 | 71 | class drop_index final : public base { 72 | std::string index_name; 73 | 74 | void exec() override; 75 | 76 | public: 77 | explicit drop_index(std::string &index_name) : index_name(index_name) {} 78 | }; 79 | 80 | class insert_table final : public base { 81 | std::string table_name; 82 | std::vector insert_list; 83 | 84 | 85 | void exec() override; 86 | 87 | public: 88 | insert_table(std::string& table_name, 89 | std::vector& insert_list) : 90 | table_name(table_name), insert_list(insert_list) {} 91 | }; 92 | 93 | class select_table final : public base { 94 | std::vector attribute_list; 95 | std::string table_name; 96 | std::vector condition_list; 97 | 98 | 99 | void exec() override; 100 | 101 | public: 102 | select_table(std::vector& attribute_list, 103 | std::string& table_name, 104 | std::vector& condition_list) : 105 | attribute_list(attribute_list), table_name(table_name), condition_list(condition_list) {} 106 | }; 107 | 108 | class use_database final : public base { 109 | std::string database_name; 110 | 111 | void exec() override { 112 | std::cout << "currently not supported"; // TODO 113 | } 114 | 115 | public: 116 | explicit use_database(std::string &database_name) : database_name(database_name) {} 117 | }; 118 | 119 | class execfile final : public base { 120 | std::string filename; 121 | void exec() override; 122 | 123 | public: 124 | explicit execfile(std::string &filename) : filename(filename) {} 125 | }; 126 | 127 | class exit final : public base { 128 | void exec() override; 129 | }; 130 | } 131 | 132 | #endif //MINISQL_API_H 133 | -------------------------------------------------------------------------------- /db/api/utils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "utils.h" 4 | 5 | #include "../CatalogManager/CatalogManager.h" 6 | #include "../utils/exception.h" 7 | 8 | extern CatalogManager cat_mgt; 9 | 10 | namespace api { 11 | void throw_on_table_exist(std::string &table_name) { 12 | if (cat_mgt.TableExist(table_name)) { 13 | throw sql_exception(200, "api", "table \'" + table_name + "\' exists"); 14 | } 15 | } 16 | void throw_on_table_not_exist(std::string &table_name) { 17 | if (!cat_mgt.TableExist(table_name)) { 18 | throw sql_exception(201, "api", "table \'" + table_name + "\' doesn't exist"); 19 | } 20 | } 21 | 22 | void print_select_result_seperator(const std::vector &col_max_length) { 23 | using namespace std; 24 | const auto size = col_max_length.size(); 25 | cout << " +-"; 26 | for (auto i = 0; i < size; i++) { 27 | cout << std::setiosflags(ios::left) 28 | << std::setw(static_cast(col_max_length.at(i))) 29 | << std::setfill('-') 30 | <<"-"; 31 | if (i < size - 1) { 32 | cout << "-+-"; 33 | } 34 | } 35 | cout << "-+ "; 36 | cout << endl; 37 | } 38 | 39 | void print_select_result(const std::vector&attribute_names, const result &res) { 40 | using namespace std; 41 | const auto size = attribute_names.size(); 42 | 43 | if (res.row.empty()) { 44 | return; 45 | } 46 | 47 | auto col_max_length = vector(size, 0); 48 | 49 | for (auto i = 0; i < size; i++) { 50 | col_max_length.at(i) = std::max(attribute_names.at(i).size(), col_max_length.at(i)); 51 | } 52 | for (auto const &row : res.row) { 53 | for (auto i = 0; i < size; i++) { 54 | col_max_length.at(i) = std::max(row.col.at(i).size(), col_max_length.at(i)); 55 | } 56 | } 57 | 58 | { 59 | print_select_result_seperator(col_max_length); 60 | cout << " | "; 61 | for (auto i = 0; i < size; i++) { 62 | cout << setiosflags(ios::left) 63 | << setw(static_cast(col_max_length.at(i))) 64 | << std::setfill(' ') 65 | << attribute_names.at(i); 66 | cout << " | "; 67 | } 68 | cout << endl; 69 | print_select_result_seperator(col_max_length); 70 | } 71 | 72 | { 73 | for (auto const &row : res.row) { 74 | cout << " | "; 75 | for (auto i = 0; i < size; i++) { 76 | cout << setiosflags(ios::left) 77 | << setw(static_cast(col_max_length.at(i))) 78 | << std::setfill(' ') 79 | << row.col.at(i); 80 | cout << " | "; 81 | } 82 | cout << endl; 83 | } 84 | print_select_result_seperator(col_max_length); 85 | } 86 | } 87 | 88 | void validate_condition(const macro::table &table, const std::vector &condition) { 89 | for (const auto &i : condition) { 90 | // auto iter = std::find(table.attribute_names.begin(), table.attribute_names.end(), i.attribute_name); 91 | auto iter = table.attribute_names.begin(); 92 | for(; iter != table.attribute_names.end(); ++iter) { 93 | if (*iter == i.attribute_name) { 94 | break; 95 | } 96 | } 97 | 98 | if (iter == table.attribute_names.end()) { 99 | throw sql_exception(204, "api", "attribute \'" + i.attribute_name + "\' not found"); 100 | } 101 | if (table.attribute_type.at(iter - table.attribute_names.begin()).type != i.value.sql_type.type) { 102 | throw sql_exception(203, "api", 103 | "attribute \'" + i.attribute_name + "\' type error: except " 104 | + return_value_name(table.attribute_type.at(iter - table.attribute_names.begin()).type) 105 | + " , got " 106 | + return_value_name(i.value.sql_type.type)); 107 | } 108 | } 109 | } 110 | 111 | std::string return_value_name(value_type v) { 112 | switch (v) { 113 | case value_type::CHAR: 114 | return "CHAR"; 115 | case value_type::INT: 116 | return "INT"; 117 | case value_type::FLOAT: 118 | return "FLOAT"; 119 | } 120 | } 121 | 122 | result convert_sql_tuple_to_result( 123 | const std::vector &attrTable, 124 | const std::vector &attrFetch, 125 | const sql_tuple& tup 126 | ) { 127 | auto result_to_return = result(); 128 | result_to_return.row.push_back(tup.fetchRow(attrTable, attrFetch)); 129 | return result_to_return; 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /db/api/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_API_UTILS_H 2 | #define MINISQL_API_UTILS_H 3 | 4 | #include "../macro.h" 5 | #include 6 | 7 | namespace api { 8 | void throw_on_table_exist(std::string &); 9 | void throw_on_table_not_exist(std::string &); 10 | void print_select_result_seperator(const std::vector &); 11 | void print_select_result(const std::vector&, const result &); 12 | void validate_condition(const macro::table &, const std::vector &); 13 | std::string return_value_name(value_type); 14 | result convert_sql_tuple_to_result( 15 | const std::vector &, 16 | const std::vector &, 17 | const sql_tuple& tup 18 | ); 19 | } 20 | 21 | #endif //MINISQL_API_UTILS_H 22 | -------------------------------------------------------------------------------- /db/external_main/external_main.cpp: -------------------------------------------------------------------------------- 1 | extern "C" 2 | { 3 | #include "external_main.h" 4 | } 5 | 6 | #include "../interpreter/interpreter.h" 7 | #include "../RecordManager/RecordManager.h" 8 | #include "../CatalogManager/CatalogManager.h" 9 | #include "../IndexManager/IndexManager.h" 10 | 11 | RecordManager rec_mgt; 12 | CatalogManager cat_mgt; 13 | BufferManager buf_mgt; 14 | IndexManager idx_mgt; 15 | 16 | // int main() { 17 | // std::cout << "Welcome to the MiniSQL monitor. Commands end with ;" << std::endl; 18 | // std::cout << "Version: v1.0-beta.1" << std::endl; 19 | // std::cout << "Copyright (c) 2021, Cupcake, Enzymii, L1ght and RalXYZ." << std::endl; 20 | // 21 | // cat_mgt.LoadFromFile(); 22 | // idx_mgt.load(); 23 | // interpret_entrance(); 24 | // } 25 | 26 | libminisql_resp external_main(const char *str) { 27 | 28 | // redirect cout to buffer 29 | std::stringstream buffer; 30 | auto old = std::cout.rdbuf(); 31 | std::cout.rdbuf(buffer.rdbuf()); 32 | 33 | auto code = sql_execution(str); 34 | 35 | // redirect cout back 36 | std::cout.rdbuf(old); 37 | 38 | // get buffer content 39 | auto buffer_content = buffer.str(); 40 | auto buffer_c_str = buffer_content.c_str(); 41 | auto res = reinterpret_cast(malloc(strlen(buffer_c_str) + 1)); 42 | strcpy(res, buffer_c_str); 43 | 44 | // auto code = 0; // TODO: set status code during execution 45 | return libminisql_resp{ code, res }; 46 | } 47 | 48 | char* get_table_names() { 49 | auto table_names = cat_mgt.get_table_names(); 50 | auto reduced = std::string(); 51 | for(auto x: table_names) { 52 | if(reduced.size() > 0) { 53 | reduced += "," + x; 54 | } else { 55 | reduced += x; 56 | } 57 | } 58 | 59 | auto reduced_c_str = reduced.c_str(); 60 | auto res = reinterpret_cast(malloc(strlen(reduced_c_str) + 1)); 61 | strcpy(res, reduced_c_str); 62 | 63 | return res; 64 | } 65 | -------------------------------------------------------------------------------- /db/external_main/external_main.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by enzymii on 4/6/22. 3 | // 4 | 5 | #ifndef MINISQL_MAIN_H 6 | #define MINISQL_MAIN_H 7 | 8 | typedef struct { 9 | int code; 10 | char *msg; 11 | } libminisql_resp; 12 | 13 | libminisql_resp external_main(const char *str); 14 | char* get_table_names(); 15 | 16 | #endif // MINISQL_MAIN_H 17 | -------------------------------------------------------------------------------- /db/go-svc/README.md: -------------------------------------------------------------------------------- 1 | # Golang Service Shell 2 | 3 | This component is coded in Golang. It starts an HTTP service, and communicate with Zookeeper. 4 | 5 | ## Response to master 6 | 7 | The response struct is defined as: 8 | 9 | ```go 10 | type Response struct { 11 | Code int `json:"code"` 12 | Msg string `json:"msg"` 13 | } 14 | ``` 15 | 16 | Here is a table, which shows the meaning of each code: 17 | 18 | | Status Code | Meaning | 19 | | ----------- | ------------------------------------------------------------ | 20 | | -1 | Error occurs when executing SQL statement | 21 | | 0 | Success executing SQL statement | 22 | | 1 | Successfully exit because of the execution of EXIT statement | 23 | 24 | ## Report table message to Zookeeper 25 | 26 | We connect with Zookeeper via a golang zk client. Then, we create a ephemeral node, and put the table info in it: 27 | 28 | ```go 29 | // get args from command line args 30 | regionName := os.Args[1] 31 | serverAddr := os.Args[2] 32 | apiPort := os.Args[3] 33 | 34 | // connect to zk client 35 | conn, _, err := zk.Connect([]string{serverAddr}, time.Second) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | // get table names via CGO interface 41 | tableNamesC := C.get_table_names() 42 | tableNames := C.GoString(tableNamesC) 43 | C.free(unsafe.Pointer(tableNamesC)) 44 | 45 | // register zk ephemeral node 46 | res, err := conn.Create("/db/"+regionName, []byte(tableNames), zk.FlagEphemeral, zk.WorldACL(zk.PermAll)) 47 | ``` 48 | -------------------------------------------------------------------------------- /db/go-svc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | #cgo CFLAGS: -I./ 5 | #cgo LDFLAGS: -L${SRCDIR}/../ -lminisql 6 | #include "../external_main/external_main.h" 7 | #include 8 | */ 9 | import "C" 10 | import ( 11 | "fmt" 12 | "net/http" 13 | "os" 14 | "time" 15 | "unsafe" 16 | 17 | "github.com/go-zookeeper/zk" 18 | "github.com/labstack/echo/v4" 19 | ) 20 | 21 | type Request struct { 22 | Command string `json:"command"` 23 | } 24 | 25 | type Response struct { 26 | Code int `json:"code"` 27 | Msg string `json:"msg"` 28 | } 29 | 30 | func main() { 31 | regionName := os.Args[1] 32 | serverAddr := os.Args[2] 33 | apiPort := os.Args[3] 34 | 35 | conn, _, err := zk.Connect([]string{serverAddr}, time.Second) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | tableNamesC := C.get_table_names() 41 | tableNames := C.GoString(tableNamesC) 42 | C.free(unsafe.Pointer(tableNamesC)) 43 | 44 | fmt.Println(tableNames) 45 | 46 | res, err := conn.Create("/db/"+regionName, []byte(tableNames), zk.FlagEphemeral, zk.WorldACL(zk.PermAll)) 47 | if err != nil { 48 | panic(err) 49 | } 50 | 51 | fmt.Println("created node:", res) 52 | 53 | e := echo.New() 54 | channel := make(chan int, 1) 55 | e.POST("/", func(c echo.Context) error { 56 | // mutex 57 | channel <- 1 58 | defer func() { <-channel }() 59 | 60 | var request Request 61 | if err := c.Bind(&request); err != nil { 62 | return c.JSON(http.StatusBadRequest, Response{ 63 | Code: -1, 64 | Msg: err.Error(), 65 | }) 66 | } 67 | 68 | res := C.external_main(C.CString(request.Command)) 69 | 70 | var response Response 71 | response.Code = int(res.code) 72 | response.Msg = C.GoString(res.msg) 73 | C.free(unsafe.Pointer(res.msg)) 74 | 75 | return c.JSON(http.StatusOK, response) 76 | }) 77 | e.Logger.Fatal(e.Start(":" + apiPort)) 78 | } 79 | -------------------------------------------------------------------------------- /db/go-svc/placeholder.cpp: -------------------------------------------------------------------------------- 1 | // this file tells cgo that this is a cpp project 2 | // do not remove this file 3 | -------------------------------------------------------------------------------- /db/go.mod: -------------------------------------------------------------------------------- 1 | module minisql 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/go-zookeeper/zk v1.0.2 7 | github.com/labstack/echo/v4 v4.7.2 // indirect 8 | github.com/labstack/gommon v0.3.1 // indirect 9 | github.com/mattn/go-colorable v0.1.11 // indirect 10 | github.com/mattn/go-isatty v0.0.14 // indirect 11 | github.com/valyala/bytebufferpool v1.0.0 // indirect 12 | github.com/valyala/fasttemplate v1.2.1 // indirect 13 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect 14 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect 15 | golang.org/x/sys v0.0.0-20211103235746-7861aae1554b // indirect 16 | golang.org/x/text v0.3.7 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /db/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= 4 | github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= 5 | github.com/labstack/echo/v4 v4.7.2 h1:Kv2/p8OaQ+M6Ex4eGimg9b9e6icoxA42JSlOR3msKtI= 6 | github.com/labstack/echo/v4 v4.7.2/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks= 7 | github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o= 8 | github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= 9 | github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= 10 | github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 11 | github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= 12 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 15 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 16 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 17 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 18 | github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= 19 | github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 20 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= 21 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 22 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= 23 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 24 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 25 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 26 | golang.org/x/sys v0.0.0-20211103235746-7861aae1554b h1:1VkfZQv42XQlA/jchYumAnv1UPo6RgF9rJFkTgZIxO4= 27 | golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 28 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 29 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 31 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 32 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 33 | -------------------------------------------------------------------------------- /db/interpreter/README.md: -------------------------------------------------------------------------------- 1 | # Interpreter 2 | 3 | The interpreter is the entrance of every SQL query loop. It gets the user input, passes the user input to lexer and parser, and gets the AST (abstract syntax tree), which stores in a C++ object. 4 | 5 | | Procedure | Input | Output | Used Tools | 6 | | :-------: | :-------: | :-----: | :-----: | 7 | | read user input | (command line input) | a null-terminated byte string | GNU Readline | 8 | | lexer | a null-terminated byte string | a null-terminated byte string with tokens tagged onto it | Flex | 9 | | parser | a null-terminated byte string with tokens tagged onto it | an AST in the C++ object | GNU Bison | 10 | 11 | `Flex` and `Bison` has their own syntax. `Flex` constructs relations between string and token, while `Bison` matches the SQL statements to syntaxes, which are defined recursively in `Bison` source code file. 12 | `Flex` and `Bison` needs to be translated to C++ before the compilation progress starts. We write `CMake` rules to check if there exists the required tools and libraries, and will trigger an error if some necessary dependency cannot be found. We chose `Cmake` because it is a cross-platform building solution. 13 | 14 | Since this is a command-line program, various command-line features are implemented using `GNU Readline`, like moving cursor through the line, pressing `TAB` button to auto-complete a keyword, and pressing up or down button to go through the input history. 15 | 16 | ## Concept explanation 17 | 18 | After getting an input string, the first thing we need to know is that what tokens are *keywords*, what are *operators*, and what are just basic string like identifiers. Here is an example: 19 | 20 | ```C++ 21 | auto a = b ; 22 | ^keyword ^identifier ^operator ^identifier ^operator 23 | ``` 24 | 25 | As the example shown above, *lexical analysis* processes the input string to a sequence of token names, identifying what each token actually is. 26 | 27 | After lexing, we still don't know how these tokens construct an expression. It feels like that we only understand the meaning of each word in an English sentence, but we cannot understand the sentence because we currently know nothing about the grammar. Then, we need to perform a *syntactic analysis*, conforming the token sequence to the rules of a formal grammar. 28 | 29 | ```C++ 30 | definition_expr: TYPE_KEYWORD IDENTIFIER OPERATOR= IDENTIFIER OPERATOR; 31 | // OK, now we know the format above is a definition expression! 32 | ``` 33 | 34 | I didn't really use pure C/C++ to implement the interpreter. Instead, I used `Flex` and `Bison` to generate lexer and parser. 35 | 36 | ## Lexical analysis (Flex) 37 | 38 | The procedure of lexing is in fact a serial of regex rules: 39 | 40 | ```flex 41 | [ \t\r\n] ; 42 | 43 | "*" return S_STAR; 44 | "'" return S_APOSTROPHE; 45 | ";" return S_SEMICOLON; 46 | "(" return S_L_BRACKETS; 47 | ")" return S_R_BRACKETS; 48 | "," return S_COMMA; 49 | "=" return S_EQUAL; 50 | "<>" return S_NOT_EQUAL; 51 | ">" return S_GREATER; 52 | ">=" return S_GREATER_EQUAL; 53 | "<" return S_LESS; 54 | "<=" return S_LESS_EQUAL; 55 | 56 | [A-Za-z0-9_.]* { return get_token(yylval.str = std::string(yytext)); } 57 | ``` 58 | 59 | 1. Do not match ` `, `\t`, `\r` and `\n` to any token. 60 | 2. Match the valid operators to the corresponding tokens. 61 | 3. Match `[A-Za-z0-9_.]*` into `get_token()`. 62 | 63 | The procedure in `get_token()` is a matching using hash table, making token matching really fast. If the string failed to match any of the token, then mark it as `V_STRING`, representing a simple string. 64 | Be aware that before trying to match with a keyword, the string has been converted to down-case. This means that the keywords are case-insensitive. 65 | 66 | For an example, after the procedure mentioned above, the SQL query `SELECT * FROM foo WHERE bar <> 1;` has been converted to the following token sequence: 67 | 68 | ``` 69 | K_SELECT S_STAR K_FROM V_STRING K_WHERE V_STRING S_NOT_EQUAL V_STRING S_SEMICOLON 70 | ``` 71 | 72 | You may be puzzled that `1` has been tagged as `V_STRING`. Yes, this actually happens. In fact, the meaning of `1` could be determined by the context. In this case, the type of these "rvalues" will be confirmed during the syntactic analysis. 73 | 74 | There needs more detailed implementations and configurations to make the lexer work. Here, I will only talk about how to build the whole project along with Flex. 75 | Flex is a lexer generator. This means that Flex is not a running program that lexes your input. In fact, it has it's own grammar, and generates C language code, which is the lexer after compilation by using C compiler. This means, every time I build my project, only compiling C++ code is not enough. We must generate lexer using Flex before C compilation. A simple solution is to execute the following command before compilation: 76 | 77 | ```shell 78 | flex -o lex.yy.cc tokenizer.l 79 | ``` 80 | 81 | However, since this project is a CMake project, there definitely exists ways to integrate this preprocessing procedure into CMake. And I eventually chose this implementation: 82 | 83 | ```cmake 84 | if(FLEX_FOUND) 85 | file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/interpreter) 86 | FLEX_TARGET(MyScanner interpreter/tokenizer.l ${CMAKE_CURRENT_BINARY_DIR}/interpreter/lex.yy.cc) 87 | ADD_FLEX_DEPENDENCY(MyScanner) 88 | #[[details omitted]] 89 | endif() 90 | ``` 91 | 92 | ## Syntactic analysis (Bison) 93 | 94 | The rules of syntactic analysis are written in `*.y` file. The name-prefix of each token has a format: 95 | 96 | 1. `C` stands for compound, meaning that it is a set which contains multiple instructions. 97 | 2. `I` stands for instruction, which is a complete SQL instruction. 98 | 3. `E` stands for expression, which is a part of SQL instruction. 99 | 4. `K` stands for keyword. 100 | 5. `V` stands for value, which might be a identifier, and also might be INT, FLOAT or CHAR value. 101 | 6. `S` stands for symbol. 102 | 103 | The content of `*.y` file looks like trees: 104 | 105 | ```bison 106 | C_TOP_INPUT: C_TOP_STMT S_SEMICOLON 107 | { 108 | YYACCEPT; 109 | } 110 | ; 111 | 112 | C_TOP_STMT: C_DDL 113 | | C_DML 114 | | I_EXIT 115 | | I_QUIT 116 | | I_USE_DATABASE 117 | | I_EXECFILE 118 | | I_VACANT 119 | ; 120 | ``` 121 | 122 | `C_TOP_INPUT` is the root of the whole tree. Matching an instruction is a tree traversing process. 123 | 124 | Now, let's focus on an *instruction* node of the tree: 125 | 126 | ```bison 127 | I_SELECT_TABLE: K_SELECT E_ATTRIBUTE_LIST K_FROM V_STRING E_WHERE 128 | { 129 | query_object_ptr = std::make_unique($2, $4, $5); 130 | } 131 | ; 132 | 133 | E_WHERE: K_WHERE E_CONDITION_LIST 134 | { 135 | $$ = $2; 136 | } 137 | | E_VACANT 138 | { 139 | $$ = std::vector(); 140 | } 141 | ; 142 | ``` 143 | 144 | In each instruction node, only one thing has been done: generating a unique pointer of the derived objects of `api::base`. `($2, $4, $5)` passes three parameters to the constructor, which are the result of expression `E_ATTRIBUTE_LIST`, the value of `V_STRING` and the result of expression `E_WHERE`. `query_object_ptr`is an external global variable, which is a part of API, which will be discussed in the next chapter. 145 | Each expression will not interact with any global value. Instead, it will return values to it's ancestors, using `$$`. 146 | 147 | Bison is a parser generator. It needs to translate from `*.y` file to C language files. To interact with this parser, we need to call its functions, which is defined in the generated files. Our implementation of parser calling is: 148 | 149 | ```c++ 150 | int parse(const char *input) { 151 | YY_BUFFER_STATE bp = yy_scan_string(input); 152 | if (bp == nullptr) { 153 | std::cerr << "error on creating YY_BUFFER_STATE" << std::endl; 154 | return -1; 155 | } 156 | yy_switch_to_buffer(bp); 157 | yyparse(); // parser calling 158 | yy_delete_buffer(bp); 159 | yylex_destroy(); 160 | return 0; 161 | } 162 | ``` 163 | 164 | This procedure is trivial. It just scan the file, check validation, switch buffer, parse, and finally free buffer and lexer. 165 | 166 | To build Bison and Flex together with CMake, here is the related code in the CMakeList: 167 | 168 | ```cmake 169 | if(BISON_FOUND AND FLEX_FOUND) 170 | file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/interpreter) 171 | BISON_TARGET(MyParser interpreter/parser.y ${CMAKE_CURRENT_BINARY_DIR}/interpreter/parser.tab.cc) 172 | FLEX_TARGET(MyScanner interpreter/tokenizer.l ${CMAKE_CURRENT_BINARY_DIR}/interpreter/lex.yy.cc) 173 | ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser) 174 | include_directories("interpreter") 175 | add_executable(#[[details omitted]] ${BISON_MyParser_OUTPUTS} ${FLEX_MyScanner_OUTPUTS}) 176 | #[[details omitted]] 177 | endif() 178 | ``` 179 | 180 | ## Issues 181 | 182 | While we are using C++ as the developing language, we are relying on many cpp-specific features, like exception handling. However, if a heap space is created using `::malloc()` or `new`, if an exception has been thrown before `::free()` or `delete`, then memory leak will happen, since destructor doesn't exist in C. This means that if Flex and Bison generates C files, then we cannot use some of the C++ features. 183 | Sadly, although both of these two tools have C++ mode, but they just cannot work together when C++ mode are both switched on. The only thing I can do is to use C++ compiler to compile the C files generated by them, while avoid using some of the C++ features that might break things. 184 | It cost me loads of times to make the generated code and my source code work together properly. 185 | -------------------------------------------------------------------------------- /db/interpreter/init.sh: -------------------------------------------------------------------------------- 1 | flex -o lex.yy.cc tokenizer.l 2 | bison -d parser.y -o parser.tab.cc -------------------------------------------------------------------------------- /db/interpreter/interpreter.cpp: -------------------------------------------------------------------------------- 1 | #ifdef READLINE_FOUND 2 | #include 3 | #include 4 | extern "C" char **keyword_completion(const char *, int, int); 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../api/api.h" 13 | #include "interpreter.h" 14 | #include "../utils/utils.h" 15 | #include "../utils/exception.h" 16 | 17 | bool sig_exit = false; 18 | std::string file_to_exec; 19 | std::unique_ptr parse_exception = nullptr; 20 | std::unique_ptr query_object_ptr = nullptr; 21 | 22 | const auto PRECISION = 4; 23 | 24 | /* 25 | code: 26 | 1: exit 27 | 0: success 28 | -1: error 29 | */ 30 | int sql_execution(const char *sql) { 31 | parse_exception = nullptr; 32 | 33 | parse(sql); 34 | 35 | auto query_exit_ptr = dynamic_cast(query_object_ptr.get()); 36 | 37 | auto start = std::chrono::system_clock::now(); 38 | 39 | if (parse_exception != nullptr) { 40 | std::cout << *parse_exception; 41 | return -1; 42 | } 43 | if(query_object_ptr != nullptr) { 44 | try { 45 | query_object_ptr->exec(); 46 | } catch (sql_exception& e) { 47 | std::cout << e; 48 | return -1; 49 | } 50 | } 51 | 52 | auto end = std::chrono::system_clock::now(); 53 | 54 | std::cout << "(" 55 | << std::fixed 56 | << std::setprecision(PRECISION) 57 | << static_cast(duration_cast(end - start).count()) 58 | * std::chrono::microseconds::period::num / std::chrono::microseconds::period::den 59 | << " sec)" 60 | << std::endl; 61 | 62 | std::cout.unsetf(std::ios::fixed); 63 | std::cout.precision(6); 64 | 65 | if (query_exit_ptr != nullptr) { 66 | return 1; 67 | } 68 | return 0; 69 | } 70 | 71 | 72 | void interpret_entrance() { 73 | while (!sig_exit) { 74 | std::cout << std::endl; 75 | interpreter inter; 76 | parse_exception = nullptr; 77 | 78 | if (file_to_exec.empty()) { 79 | auto flag = sql_execution(inter.read()); 80 | if(flag == 1) { 81 | std::exit(0); 82 | } 83 | } else { // execfile 84 | std::ifstream file(file_to_exec); 85 | if (file.fail()) { 86 | std::cout << sql_exception(101, "interpreter", "file \'" + file_to_exec + "\' not found"); 87 | } else { 88 | while (!file.eof()) { 89 | interpreter inter_inner; 90 | auto flag = sql_execution(inter_inner.read(file)); 91 | if(flag == 1) { 92 | std::exit(0); 93 | } 94 | } 95 | } 96 | file_to_exec = ""; 97 | } 98 | } 99 | } 100 | 101 | const char* interpreter::read() { 102 | #ifdef READLINE_FOUND 103 | rl_attempted_completion_function = keyword_completion; 104 | if (this->str != nullptr) { 105 | std::free(this->str); 106 | this->str = nullptr; 107 | } 108 | const char *input = readline(this->start_text()); 109 | add_history(input); 110 | return input; 111 | #else 112 | while (true) { 113 | std::string src; 114 | std::cout << this->start_text(); 115 | std::getline(std::cin, src); 116 | while (src.ends_with(' ')) { 117 | src.pop_back(); 118 | } 119 | if (!first_loop) { 120 | strcat_s(this->str, "\n", SQL_QUERY_LENGTH); 121 | } 122 | strcat_s(this->str, src.c_str(), SQL_QUERY_LENGTH); 123 | if (src.length() >= 1 && *src.rbegin() == ';') { 124 | break; 125 | } 126 | first_loop = false; 127 | } 128 | return this->str; 129 | #endif 130 | } 131 | 132 | const char* interpreter::read(std::istream &is) { 133 | is.getline(this->str, SQL_QUERY_LENGTH, ';'); 134 | strcat_s(this->str, ";", SQL_QUERY_LENGTH); 135 | return this->str; 136 | } 137 | -------------------------------------------------------------------------------- /db/interpreter/interpreter.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_INTERPRETER_H 2 | #define MINISQL_INTERPRETER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "../macro.h" 9 | 10 | const int SQL_QUERY_LENGTH = 200; 11 | 12 | void interpret_entrance(); 13 | int sql_execution(const char *); 14 | 15 | extern "C" int yywrap ( void ); 16 | 17 | int yylex(); 18 | 19 | int yyparse(); 20 | 21 | int yyerror(const char *); 22 | 23 | int parse(const char *); 24 | 25 | typedef struct yystype { 26 | bool b; 27 | std::string str; 28 | std::vector insert_list; 29 | std::vector attribute_list; 30 | std::vector condition_list; 31 | condition condition_item; 32 | attribute_operator op; 33 | std::vector schema_list; 34 | schema schema_item; 35 | sql_value_type type; 36 | sql_value v; 37 | 38 | } YYSTYPE; 39 | 40 | class interpreter { 41 | bool first_loop = true; 42 | char *str = nullptr; 43 | 44 | char* start_text() const { 45 | if (first_loop) { 46 | return const_cast("MiniSQL> "); 47 | } else { 48 | return const_cast(" -> "); 49 | } 50 | } 51 | 52 | public: 53 | inline interpreter() { 54 | str = reinterpret_cast(std::malloc(SQL_QUERY_LENGTH)); 55 | str[0] = '\0'; 56 | } 57 | inline ~interpreter() { 58 | if (str != nullptr) { 59 | std::free(str); 60 | } 61 | str = nullptr; 62 | } 63 | 64 | const char* read(); 65 | const char* read(std::istream&); 66 | }; 67 | 68 | #endif //MINISQL_INTERPRETER_H 69 | -------------------------------------------------------------------------------- /db/interpreter/parser.y: -------------------------------------------------------------------------------- 1 | %{ 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "interpreter.h" 8 | #include "../api/api.h" 9 | #include "../utils/exception.h" 10 | 11 | extern std::unique_ptr parse_exception; 12 | extern std::unique_ptr query_object_ptr; 13 | 14 | bool bFlag; /* no meanings. */ 15 | 16 | extern int yylineno; 17 | extern char *yytext; 18 | inline int yyerror(const char *s) 19 | { 20 | // std::cerr << s << " on token " << yytext << std::endl; 21 | parse_exception = std::make_unique(100, "interpreter", std::string(s) + " near \'" + std::string(yytext) + "\' at line " + std::to_string(yylineno)); 22 | yywrap(); 23 | return 1; 24 | } 25 | 26 | %} 27 | 28 | %token 29 | K_DATABASE K_TABLE K_INDEX K_VALUES 30 | K_FROM K_WHERE K_ON K_INTO K_AND 31 | K_CREATE K_SELECT K_INSERT K_DELETE K_DROP 32 | K_USE K_EXIT K_QUIT K_PRIMARY K_KEY K_UNIQUE K_EXECFILE 33 | T_INT T_FLOAT T_CHAR 34 | S_APOSTROPHE S_SEMICOLON S_L_BRACKETS S_R_BRACKETS S_COMMA S_STAR 35 | S_EQUAL S_NOT_EQUAL S_GREATER S_GREATER_EQUAL S_LESS S_LESS_EQUAL 36 | 37 | %token V_STRING 38 | %type V_INSERT 39 | %type E_UNIQUE 40 | %type E_ATTRIBUTE_LIST 41 | %type E_WHERE 42 | %type E_CONDITION_LIST 43 | %type E_CONDITION 44 | %type E_OPERATOR 45 | %type E_INSERT_LIST 46 | %type E_SCHEMA_LIST 47 | %type E_SCHEMA 48 | %type E_TYPE 49 | %type E_PRIMARY_KEY 50 | 51 | %% 52 | 53 | C_TOP_INPUT: C_TOP_STMT S_SEMICOLON 54 | { 55 | YYACCEPT; 56 | } 57 | ; 58 | 59 | C_TOP_STMT: C_DDL 60 | | C_DML 61 | | I_EXIT 62 | | I_QUIT 63 | | I_USE_DATABASE 64 | | I_EXECFILE 65 | | I_VACANT 66 | ; 67 | 68 | C_DDL: I_CREATE_TABLE 69 | | I_DROP_TABLE 70 | | I_CREATE_INDEX 71 | | I_DROP_INDEX 72 | ; 73 | 74 | C_DML: I_INSERT_TABLE 75 | | I_SELECT_TABLE 76 | | I_DELETE_TABLE 77 | ; 78 | 79 | I_EXIT: K_EXIT 80 | { 81 | query_object_ptr = std::make_unique(); 82 | } 83 | ; 84 | 85 | I_QUIT: K_QUIT 86 | { 87 | query_object_ptr = std::make_unique(); 88 | } 89 | ; 90 | 91 | I_USE_DATABASE: K_USE K_DATABASE V_STRING 92 | { 93 | query_object_ptr = std::make_unique($3); 94 | } 95 | ; 96 | 97 | I_CREATE_TABLE: K_CREATE K_TABLE V_STRING S_L_BRACKETS E_SCHEMA_LIST E_PRIMARY_KEY S_R_BRACKETS 98 | { 99 | query_object_ptr = std::make_unique($3, $5, $6); 100 | } 101 | ; 102 | 103 | I_DELETE_TABLE: K_DELETE K_FROM V_STRING E_WHERE 104 | { 105 | query_object_ptr = std::make_unique($3, $4); 106 | } 107 | 108 | I_DROP_TABLE: K_DROP K_TABLE V_STRING 109 | { 110 | query_object_ptr = std::make_unique($3); 111 | } 112 | ; 113 | 114 | I_CREATE_INDEX: K_CREATE K_INDEX V_STRING K_ON V_STRING S_L_BRACKETS V_STRING S_R_BRACKETS 115 | { 116 | query_object_ptr = std::make_unique($3, $5, $7); 117 | } 118 | ; 119 | 120 | I_DROP_INDEX: K_DROP K_INDEX V_STRING 121 | { 122 | query_object_ptr = std::make_unique($3); 123 | } 124 | ; 125 | 126 | I_INSERT_TABLE: K_INSERT K_INTO V_STRING K_VALUES S_L_BRACKETS E_INSERT_LIST S_R_BRACKETS 127 | { 128 | query_object_ptr = std::make_unique($3, $6); 129 | } 130 | ; 131 | 132 | I_SELECT_TABLE: K_SELECT E_ATTRIBUTE_LIST K_FROM V_STRING E_WHERE 133 | { 134 | query_object_ptr = std::make_unique($2, $4, $5); 135 | } 136 | ; 137 | 138 | I_EXECFILE: K_EXECFILE S_APOSTROPHE V_STRING S_APOSTROPHE 139 | { 140 | query_object_ptr = std::make_unique($3); 141 | } 142 | ; 143 | 144 | I_VACANT: E_VACANT 145 | { 146 | query_object_ptr = std::make_unique(); 147 | } 148 | 149 | /******************************************************************************/ 150 | 151 | E_SCHEMA_LIST: E_SCHEMA_LIST S_COMMA E_SCHEMA 152 | { 153 | $$ = $1; 154 | $$.push_back($3); 155 | } 156 | | E_SCHEMA 157 | { 158 | $$ = std::vector(); 159 | $$.push_back($1); 160 | } 161 | ; 162 | 163 | E_SCHEMA: V_STRING E_TYPE E_UNIQUE 164 | { 165 | $$ = schema($1, $2, $3); 166 | } 167 | ; 168 | 169 | E_TYPE: T_INT 170 | { 171 | $$ = sql_value_type(value_type::INT); 172 | } 173 | | T_FLOAT 174 | { 175 | $$ = sql_value_type(value_type::FLOAT); 176 | } 177 | | T_CHAR S_L_BRACKETS V_STRING S_R_BRACKETS 178 | { 179 | $$ = sql_value_type(std::stoi($3)); // FIXME: exception 180 | } 181 | ; 182 | 183 | E_PRIMARY_KEY: S_COMMA K_PRIMARY K_KEY S_L_BRACKETS V_STRING S_R_BRACKETS 184 | { 185 | $$ = $5; 186 | } 187 | | E_VACANT 188 | { 189 | $$ = ""; 190 | } 191 | ; 192 | 193 | E_UNIQUE: K_UNIQUE 194 | { 195 | $$ = true; 196 | } 197 | | E_VACANT 198 | { 199 | $$ = false; 200 | } 201 | ; 202 | 203 | E_INSERT_LIST: E_INSERT_LIST S_COMMA V_INSERT 204 | { 205 | $$ = $1; 206 | $$.push_back($3); 207 | } 208 | | V_INSERT 209 | { 210 | $$ = std::vector(); 211 | $$.push_back($1); 212 | } 213 | ; 214 | 215 | E_ATTRIBUTE_LIST: S_STAR 216 | { 217 | $$ = std::vector(); 218 | } 219 | | E_ATTRIBUTE_LIST S_COMMA V_STRING 220 | { 221 | $$.push_back($3); 222 | } 223 | | V_STRING 224 | { 225 | $$ = std::vector(); 226 | $$.push_back($1); 227 | } 228 | ; 229 | 230 | E_WHERE: K_WHERE E_CONDITION_LIST 231 | { 232 | $$ = $2; 233 | } 234 | | E_VACANT 235 | { 236 | $$ = std::vector(); 237 | } 238 | ; 239 | 240 | E_CONDITION_LIST: E_CONDITION_LIST K_AND E_CONDITION 241 | { 242 | $$ = $1; 243 | $$.push_back($3); 244 | } 245 | | E_CONDITION 246 | { 247 | $$ = std::vector(); 248 | $$.push_back($1); 249 | } 250 | ; 251 | 252 | E_CONDITION: V_STRING E_OPERATOR V_INSERT 253 | { 254 | $$.attribute_name = $1; 255 | $$.op = $2; 256 | $$.value = $3; 257 | } 258 | 259 | E_OPERATOR: 260 | S_EQUAL {$$ = attribute_operator::EQUAL;} 261 | | S_NOT_EQUAL {$$ = attribute_operator::NOT_EQUAL;} 262 | | S_GREATER {$$ = attribute_operator::GREATER;} 263 | | S_GREATER_EQUAL {$$ = attribute_operator::GREATER_EQUAL;} 264 | | S_LESS {$$ = attribute_operator::LESS;} 265 | | S_LESS_EQUAL {$$ = attribute_operator::LESS_EQUAL;} 266 | 267 | 268 | V_INSERT: S_APOSTROPHE V_STRING S_APOSTROPHE 269 | { 270 | $$ = $2; 271 | } 272 | | V_STRING 273 | { 274 | if ($1.find('.') != std::string::npos) { 275 | $$ = sql_value(std::stof($1)); 276 | } 277 | else { 278 | try { 279 | $$ = sql_value(std::stoi($1)); 280 | } catch (std::exception) { 281 | $$ = sql_value($1); 282 | } 283 | } 284 | } 285 | ; 286 | 287 | E_VACANT: 288 | ; 289 | 290 | %% 291 | -------------------------------------------------------------------------------- /db/interpreter/readline.c: -------------------------------------------------------------------------------- 1 | #ifdef READLINE_FOUND 2 | 3 | #include 4 | 5 | #include 6 | 7 | char **keyword_completion(const char *, int, int); 8 | char *keyword_generator(const char *, int); 9 | 10 | char *keywords[] = { 11 | "database", "table", "index", "values", 12 | "from", "where", "on", "into", "and", 13 | "create", "select", "insert", "delete", "drop", 14 | "use", "exit", "quit", "primary", "key", "unique", "execfile", 15 | "int", "float", "char", 16 | NULL 17 | }; 18 | 19 | char ** 20 | keyword_completion(const char *text, int start, int end) 21 | { 22 | rl_attempted_completion_over = 1; 23 | return rl_completion_matches(text, keyword_generator); 24 | } 25 | 26 | char * 27 | keyword_generator(const char *text, int state) 28 | { 29 | static int list_index, len; 30 | char *name; 31 | 32 | if (!state) { 33 | list_index = 0; 34 | len = strlen(text); 35 | } 36 | 37 | while ((name = keywords[list_index++])) { 38 | if (strncmp(name, text, len) == 0) { 39 | return strdup(name); 40 | } 41 | } 42 | 43 | return NULL; 44 | } 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /db/interpreter/tokenizer.l: -------------------------------------------------------------------------------- 1 | %{ 2 | #include 3 | #include 4 | #include "interpreter.h" 5 | #include "parser.tab.hh" 6 | %} 7 | 8 | 9 | %% 10 | 11 | [ \t\r\n] ; 12 | 13 | "*" return S_STAR; 14 | "'" return S_APOSTROPHE; 15 | ";" return S_SEMICOLON; 16 | "(" return S_L_BRACKETS; 17 | ")" return S_R_BRACKETS; 18 | "," return S_COMMA; 19 | 20 | "=" return S_EQUAL; 21 | "<>" return S_NOT_EQUAL; 22 | ">" return S_GREATER; 23 | ">=" return S_GREATER_EQUAL; 24 | "<" return S_LESS; 25 | "<=" return S_LESS_EQUAL; 26 | 27 | database|DATABASE return K_DATABASE; 28 | table|TABLE return K_TABLE; 29 | index|INDEX return K_INDEX; 30 | values|VALUES return K_VALUES; 31 | 32 | from|FROM return K_FROM; 33 | where|WHERE return K_WHERE; 34 | on|ON return K_ON; 35 | into|INTO return K_INTO; 36 | and|AND return K_AND; 37 | 38 | create|CREATE return K_CREATE; 39 | select|SELECT return K_SELECT; 40 | insert|INSERT return K_INSERT; 41 | delete|DELETE return K_DELETE; 42 | drop|DROP return K_DROP; 43 | 44 | use|USE return K_USE; 45 | exit|EXIT return K_EXIT; 46 | quit|QUIT return K_QUIT; 47 | primary|PRIMARY return K_PRIMARY; 48 | key|KEY return K_KEY; 49 | unique|UNIQUE return K_UNIQUE; 50 | execfile|EXECFILE return K_EXECFILE; 51 | 52 | int|INT return T_INT; 53 | float|FLOAT return T_FLOAT; 54 | char|CHAR return T_CHAR; 55 | 56 | [A-Za-z0-9_.]* { 57 | yylval.str = std::string(yytext); 58 | return V_STRING; 59 | } 60 | 61 | %% 62 | 63 | int yywrap() 64 | { 65 | yy_flush_buffer(YY_CURRENT_BUFFER); 66 | { BEGIN INITIAL; } 67 | return 1; 68 | } 69 | 70 | int parse(const char *input) { 71 | YY_BUFFER_STATE bp = yy_scan_string(input); 72 | if (bp == nullptr) { 73 | std::cerr << "error on creating YY_BUFFER_STATE" << std::endl; 74 | return -1; 75 | } 76 | yy_switch_to_buffer(bp); 77 | yyparse(); 78 | yy_delete_buffer(bp); 79 | yylex_destroy(); 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /db/macro.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_MACRO_H 2 | #define MINISQL_MACRO_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | enum class value_type { 12 | INT, 13 | CHAR, //string 14 | FLOAT, 15 | }; 16 | 17 | struct sql_value_type { 18 | value_type type; 19 | uint8_t length = 0; 20 | bool primary = false, unique = false; 21 | uint8_t size() const{ 22 | switch(type){ 23 | case value_type::INT: 24 | return sizeof(int); 25 | case value_type::FLOAT: 26 | return sizeof(float); 27 | case value_type::CHAR: 28 | return length + 1; 29 | } 30 | } 31 | 32 | sql_value_type() = default; 33 | sql_value_type(value_type type) : type(type) {} 34 | sql_value_type(uint8_t length) : type(value_type::CHAR), length(length) {} 35 | }; 36 | 37 | struct sql_value { 38 | sql_value_type sql_type; 39 | int sql_int; 40 | float sql_float; 41 | std::string sql_str; 42 | 43 | sql_value() = default; 44 | sql_value(int i) : sql_int(i), sql_type(value_type::INT) {}; 45 | sql_value(float f) : sql_float(f), sql_type(value_type::FLOAT) {}; 46 | sql_value(std::string& str) : sql_str(str), sql_type(value_type::CHAR) {}; 47 | 48 | void reset() { 49 | sql_int = 0; 50 | sql_float = 0; 51 | sql_str.clear(); 52 | } 53 | 54 | std::string toStr() const { 55 | switch (sql_type.type) { 56 | case value_type::INT: 57 | return std::to_string(sql_int); 58 | case value_type::FLOAT: 59 | return std::to_string(sql_float); 60 | case value_type::CHAR: 61 | return this->sql_str; 62 | } 63 | } 64 | 65 | bool operator<(const sql_value &e) const { 66 | switch (sql_type.type) { 67 | case value_type::INT: 68 | return sql_int < e.sql_int; 69 | case value_type::FLOAT: 70 | return sql_float < e.sql_float; 71 | case value_type::CHAR: 72 | return sql_str < e.sql_str; 73 | default: 74 | throw std::runtime_error("Undefined Type!"); 75 | } 76 | } 77 | 78 | bool operator==(const sql_value &e) const { 79 | switch (sql_type.type) { 80 | case value_type::INT: 81 | return sql_int == e.sql_int; 82 | case value_type::FLOAT: 83 | return sql_float == e.sql_float; 84 | case value_type::CHAR: 85 | return sql_str == e.sql_str; 86 | default: 87 | throw std::runtime_error("Undefined Type!"); 88 | } 89 | } 90 | 91 | bool operator!=(const sql_value &e) const { return !operator==(e); } 92 | 93 | bool operator>(const sql_value &e) const { return !operator<(e) && operator!=(e); } 94 | 95 | bool operator<=(const sql_value &e) const { return operator<(e) || operator==(e); } 96 | 97 | bool operator>=(const sql_value &e) const { return !operator<(e); } 98 | }; 99 | 100 | struct schema { 101 | std::string name; 102 | sql_value_type type; 103 | bool unique; 104 | 105 | schema() = default; 106 | schema(std::string& name, sql_value_type& type, bool unique) : name(name), type(type), unique(unique) {} 107 | }; 108 | 109 | enum class attribute_operator { 110 | EQUAL, 111 | NOT_EQUAL, 112 | GREATER, 113 | GREATER_EQUAL, 114 | LESS, 115 | LESS_EQUAL, 116 | }; 117 | 118 | struct condition { 119 | std::string attribute_name; 120 | attribute_operator op; 121 | sql_value value; //FIXME!!! chang type to sql_value 122 | 123 | condition() = default; 124 | condition(std::string& attribute_name, attribute_operator op, sql_value& value) : 125 | attribute_name(attribute_name), 126 | op(op), 127 | value(value) {} 128 | 129 | bool test(const sql_value &e) const { 130 | switch (op) { 131 | case attribute_operator::EQUAL: 132 | return e == value; 133 | case attribute_operator::NOT_EQUAL: 134 | return e != value; 135 | case attribute_operator::GREATER: 136 | return e > value; 137 | case attribute_operator::GREATER_EQUAL: 138 | return e >= value; 139 | case attribute_operator::LESS: 140 | return e < value; 141 | case attribute_operator::LESS_EQUAL: 142 | return e <= value; 143 | default: 144 | std::cerr << "Undefined condition width cond !" << std::endl; 145 | } 146 | } 147 | }; 148 | 149 | struct row { 150 | std::vector col; 151 | }; 152 | 153 | struct result { 154 | std::vector row; 155 | }; 156 | 157 | struct sql_tuple { 158 | std::vector element; 159 | 160 | /* 161 | * attrTable: all columns 162 | * attrFetch: columns to be selected 163 | */ 164 | row fetchRow(const std::vector &attrTable, const std::vector &attrFetch) const { 165 | row row; 166 | bool attrFound; 167 | row.col.reserve(attrFetch.size()); 168 | for (auto fetch : attrFetch) { 169 | attrFound = false; 170 | for (int i = 0; i < attrTable.size(); i++) { 171 | if (fetch == attrTable[i]) { 172 | row.col.push_back(element[i].toStr()); 173 | attrFound = true; 174 | break; 175 | } 176 | } 177 | if (!attrFound) { 178 | std::cerr << "Undefined attr in row fetching!!" << std::endl; 179 | } 180 | } 181 | return row; 182 | } 183 | 184 | // const sql_value_type &fetchElement(const std::vector &attrTable, const std::string &attrFetch) const { 185 | // for (int i = 0; i < attrTable.size(); i++) { 186 | // if (attrFetch == attrTable[i]) { 187 | // return element[i]; 188 | // } 189 | // } 190 | // std::cerr << "Undefined attr in element fetching from tuple!!" << std::endl; 191 | // } 192 | }; 193 | 194 | namespace macro { 195 | 196 | const int BlockSize = 128; 197 | const int MaxBlocks = 4; 198 | const int MaxFiles = 6; 199 | 200 | inline std::string tableFile(const std::string &tableName) { return tableName + ".tb"; } 201 | 202 | struct table { 203 | table() {}; 204 | table(std::string table_name, int attr_cnt) 205 | :name(table_name), attribute_cnt(attr_cnt){}; 206 | /* table(const table &T) : name(T.name), attribute_cnt(T.attribute_cnt), record_len(T.record_len), 207 | record_cnt(T.record_cnt), size(T.size), attribute_type(T.attribute_type), attribute_names(T.attribute_names), 208 | indexNames(T.indexNames) {};*/ 209 | 210 | std::string name; 211 | int attribute_cnt, record_len, record_cnt, size; 212 | 213 | std::vector attribute_type; 214 | std::vector attribute_names; 215 | // for index, first stands for attr name, second stands for index name. 216 | std::vector> index; 217 | 218 | friend std::ostream &operator<<(std::ostream &os, const table &tab) { 219 | os << "name: " << tab.name << " attribute_cnt: " << tab.attribute_cnt << " record_len: " << tab.record_len 220 | << " record_cnt: " << tab.record_cnt << " size: " << tab.size 221 | << " attribute_names: " << tab.attribute_names.size(); 222 | return os; 223 | } 224 | }; 225 | } 226 | 227 | #endif //MINISQL_MACRO_H 228 | -------------------------------------------------------------------------------- /db/main.cpp: -------------------------------------------------------------------------------- 1 | #include "../interpreter/interpreter.h" 2 | #include "../RecordManager/RecordManager.h" 3 | #include "../CatalogManager/CatalogManager.h" 4 | #include "../IndexManager/IndexManager.h" 5 | 6 | RecordManager rec_mgt; 7 | CatalogManager cat_mgt; 8 | BufferManager buf_mgt; 9 | IndexManager idx_mgt; 10 | 11 | int main() { 12 | std::cout << "Welcome to the MiniSQL monitor. Commands end with ;" << std::endl; 13 | std::cout << "Version: v1.0-beta.1" << std::endl; 14 | std::cout << "Copyright (c) 2021, Cupcake, Enzymii, L1ght and RalXYZ." << std::endl; 15 | 16 | interpret_entrance(); 17 | } -------------------------------------------------------------------------------- /db/utils/exception.cpp: -------------------------------------------------------------------------------- 1 | #include "exception.h" 2 | 3 | std::ostream& operator<<(std::ostream& os, const sql_exception& e) { 4 | os << "ERROR " << e.code << " from <" << e.module_name << ">: " << e.detail; 5 | return os; 6 | } 7 | -------------------------------------------------------------------------------- /db/utils/exception.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_EXCEPTION_H 2 | #define MINISQL_EXCEPTION_H 3 | 4 | #include 5 | #include 6 | 7 | class sql_exception : public std::exception { 8 | std::string module_name; 9 | std::string detail; 10 | 11 | public: 12 | unsigned int code; 13 | 14 | sql_exception() = default; 15 | sql_exception(unsigned int code, std::string& module_name, std::string& detail) : 16 | code(code), module_name(module_name), detail(detail) {} 17 | 18 | sql_exception(unsigned int code, std::string&& module_name, std::string&& detail) : 19 | code(code), module_name(module_name), detail(detail) {} 20 | 21 | inline const char* what() const noexcept override { 22 | return detail.c_str(); 23 | } 24 | 25 | friend std::ostream& operator<<(std::ostream& os, const sql_exception& e); 26 | }; 27 | 28 | #endif //MINISQL_EXCEPTION_H 29 | -------------------------------------------------------------------------------- /db/utils/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include "exception.h" 3 | 4 | char* strcat_s(char* dest, const char* src, std::size_t max_length) { 5 | if (std::strlen(dest) + std::strlen(src) > max_length) { 6 | throw sql_exception(900, "utilities", "input buffer overflow"); 7 | } 8 | 9 | return std::strcat(dest, src); 10 | } 11 | 12 | std::string generate_table_pk_name(const std::string& table_name) { 13 | return table_name + ".pk"; 14 | } 15 | -------------------------------------------------------------------------------- /db/utils/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef MINISQL_UTILS_H 2 | #define MINISQL_UTILS_H 3 | 4 | #include "../macro.h" 5 | 6 | char* strcat_s(char*, const char*, std::size_t); 7 | 8 | std::string generate_table_pk_name(const std::string&); 9 | 10 | #endif //MINISQL_UTILS_H 11 | -------------------------------------------------------------------------------- /master/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | 35 | ### Configuration ### 36 | application.yaml -------------------------------------------------------------------------------- /master/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BMS-2021/MiniSQL/f3b3644aa7ba6cccf51309743ee77354514fec5c/master/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /master/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar 3 | -------------------------------------------------------------------------------- /master/README.md: -------------------------------------------------------------------------------- 1 | # MiniSQL Master 2 | 3 | **Disclaimer: This project is the final project of a course, which can only be considered as a toy. It shows some undergraduates' ideas on how to build a proper database. Suggestions are welcomed.** 4 | 5 | MiniSQL master manages all region servers. It starts region servers, keep track on how tables distribute on different regions, load balancing request and so on. 6 | 7 |
8 | 9 |
10 | 11 | 12 | There are three services in master. 13 | 14 | *Controller Service* is responsible for receiving and pre-processing HTTP requests, of which there are two types. 15 | - The first one is `GET` cache request, where the client gets the cache information from the master. Since the master will synchronize the region and table correspondence in real time via a callback function, this request is actually taking the mapping table. 16 | - The second type of request is `POST` statement request. The scenario is that the client sends a request to the master with side effects. As the request body contains table information, the Master master tries to find which region it should hit with the request, again by looking up the mapping table. The structure of the region response is described in `db/go-svc`. 17 | 18 | *Cluster Service* is responsible for region start/stop, disaster recovery and callback function registration. In short, everything related to cluster management is coded here. In a testing environment, this service is also responsible for starting a Zookeeper cluster; regardless of the environment, a Zookeeper Curator Client is always maintained here, which pulls up the region by constructing and executing command-line instructions when the cluster is started or a disaster recovery takes place. On the region side, the master monitors the state of the region by registering a callback function with Curator. The callback function monitors the directory for updates. As soon as an update is made to the Zookeeper directory (typically a new table), the master synchronises the update to the data structure of the region where the table is stored in its own memory. 19 | 20 | *Config Service* is responsible for loading configuration files. Loading configuration files is a native feature of the Spring framework that allows configuration information to be stored in specified class member variables. The configuration file we store includes four fields `zkUrl`, `zkPathName`, `regionPort`, `shellPath`, which are metadata that related to all regions and Zookeeper. 21 | -------------------------------------------------------------------------------- /master/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /usr/local/etc/mavenrc ] ; then 40 | . /usr/local/etc/mavenrc 41 | fi 42 | 43 | if [ -f /etc/mavenrc ] ; then 44 | . /etc/mavenrc 45 | fi 46 | 47 | if [ -f "$HOME/.mavenrc" ] ; then 48 | . "$HOME/.mavenrc" 49 | fi 50 | 51 | fi 52 | 53 | # OS specific support. $var _must_ be set to either true or false. 54 | cygwin=false; 55 | darwin=false; 56 | mingw=false 57 | case "`uname`" in 58 | CYGWIN*) cygwin=true ;; 59 | MINGW*) mingw=true;; 60 | Darwin*) darwin=true 61 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 62 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 63 | if [ -z "$JAVA_HOME" ]; then 64 | if [ -x "/usr/libexec/java_home" ]; then 65 | export JAVA_HOME="`/usr/libexec/java_home`" 66 | else 67 | export JAVA_HOME="/Library/Java/Home" 68 | fi 69 | fi 70 | ;; 71 | esac 72 | 73 | if [ -z "$JAVA_HOME" ] ; then 74 | if [ -r /etc/gentoo-release ] ; then 75 | JAVA_HOME=`java-config --jre-home` 76 | fi 77 | fi 78 | 79 | if [ -z "$M2_HOME" ] ; then 80 | ## resolve links - $0 may be a link to maven's home 81 | PRG="$0" 82 | 83 | # need this for relative symlinks 84 | while [ -h "$PRG" ] ; do 85 | ls=`ls -ld "$PRG"` 86 | link=`expr "$ls" : '.*-> \(.*\)$'` 87 | if expr "$link" : '/.*' > /dev/null; then 88 | PRG="$link" 89 | else 90 | PRG="`dirname "$PRG"`/$link" 91 | fi 92 | done 93 | 94 | saveddir=`pwd` 95 | 96 | M2_HOME=`dirname "$PRG"`/.. 97 | 98 | # make it fully qualified 99 | M2_HOME=`cd "$M2_HOME" && pwd` 100 | 101 | cd "$saveddir" 102 | # echo Using m2 at $M2_HOME 103 | fi 104 | 105 | # For Cygwin, ensure paths are in UNIX format before anything is touched 106 | if $cygwin ; then 107 | [ -n "$M2_HOME" ] && 108 | M2_HOME=`cygpath --unix "$M2_HOME"` 109 | [ -n "$JAVA_HOME" ] && 110 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 111 | [ -n "$CLASSPATH" ] && 112 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 113 | fi 114 | 115 | # For Mingw, ensure paths are in UNIX format before anything is touched 116 | if $mingw ; then 117 | [ -n "$M2_HOME" ] && 118 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 119 | [ -n "$JAVA_HOME" ] && 120 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 121 | fi 122 | 123 | if [ -z "$JAVA_HOME" ]; then 124 | javaExecutable="`which javac`" 125 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 126 | # readlink(1) is not available as standard on Solaris 10. 127 | readLink=`which readlink` 128 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 129 | if $darwin ; then 130 | javaHome="`dirname \"$javaExecutable\"`" 131 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 132 | else 133 | javaExecutable="`readlink -f \"$javaExecutable\"`" 134 | fi 135 | javaHome="`dirname \"$javaExecutable\"`" 136 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 137 | JAVA_HOME="$javaHome" 138 | export JAVA_HOME 139 | fi 140 | fi 141 | fi 142 | 143 | if [ -z "$JAVACMD" ] ; then 144 | if [ -n "$JAVA_HOME" ] ; then 145 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 146 | # IBM's JDK on AIX uses strange locations for the executables 147 | JAVACMD="$JAVA_HOME/jre/sh/java" 148 | else 149 | JAVACMD="$JAVA_HOME/bin/java" 150 | fi 151 | else 152 | JAVACMD="`\\unset -f command; \\command -v java`" 153 | fi 154 | fi 155 | 156 | if [ ! -x "$JAVACMD" ] ; then 157 | echo "Error: JAVA_HOME is not defined correctly." >&2 158 | echo " We cannot execute $JAVACMD" >&2 159 | exit 1 160 | fi 161 | 162 | if [ -z "$JAVA_HOME" ] ; then 163 | echo "Warning: JAVA_HOME environment variable is not set." 164 | fi 165 | 166 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 167 | 168 | # traverses directory structure from process work directory to filesystem root 169 | # first directory with .mvn subdirectory is considered project base directory 170 | find_maven_basedir() { 171 | 172 | if [ -z "$1" ] 173 | then 174 | echo "Path not specified to find_maven_basedir" 175 | return 1 176 | fi 177 | 178 | basedir="$1" 179 | wdir="$1" 180 | while [ "$wdir" != '/' ] ; do 181 | if [ -d "$wdir"/.mvn ] ; then 182 | basedir=$wdir 183 | break 184 | fi 185 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 186 | if [ -d "${wdir}" ]; then 187 | wdir=`cd "$wdir/.."; pwd` 188 | fi 189 | # end of workaround 190 | done 191 | echo "${basedir}" 192 | } 193 | 194 | # concatenates all lines of a file 195 | concat_lines() { 196 | if [ -f "$1" ]; then 197 | echo "$(tr -s '\n' ' ' < "$1")" 198 | fi 199 | } 200 | 201 | BASE_DIR=`find_maven_basedir "$(pwd)"` 202 | if [ -z "$BASE_DIR" ]; then 203 | exit 1; 204 | fi 205 | 206 | ########################################################################################## 207 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 208 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 209 | ########################################################################################## 210 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 211 | if [ "$MVNW_VERBOSE" = true ]; then 212 | echo "Found .mvn/wrapper/maven-wrapper.jar" 213 | fi 214 | else 215 | if [ "$MVNW_VERBOSE" = true ]; then 216 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 217 | fi 218 | if [ -n "$MVNW_REPOURL" ]; then 219 | jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 220 | else 221 | jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 222 | fi 223 | while IFS="=" read key value; do 224 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 225 | esac 226 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 227 | if [ "$MVNW_VERBOSE" = true ]; then 228 | echo "Downloading from: $jarUrl" 229 | fi 230 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 231 | if $cygwin; then 232 | wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` 233 | fi 234 | 235 | if command -v wget > /dev/null; then 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Found wget ... using wget" 238 | fi 239 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 240 | wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 241 | else 242 | wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 243 | fi 244 | elif command -v curl > /dev/null; then 245 | if [ "$MVNW_VERBOSE" = true ]; then 246 | echo "Found curl ... using curl" 247 | fi 248 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 249 | curl -o "$wrapperJarPath" "$jarUrl" -f 250 | else 251 | curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f 252 | fi 253 | 254 | else 255 | if [ "$MVNW_VERBOSE" = true ]; then 256 | echo "Falling back to using Java to download" 257 | fi 258 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 259 | # For Cygwin, switch paths to Windows format before running javac 260 | if $cygwin; then 261 | javaClass=`cygpath --path --windows "$javaClass"` 262 | fi 263 | if [ -e "$javaClass" ]; then 264 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 265 | if [ "$MVNW_VERBOSE" = true ]; then 266 | echo " - Compiling MavenWrapperDownloader.java ..." 267 | fi 268 | # Compiling the Java class 269 | ("$JAVA_HOME/bin/javac" "$javaClass") 270 | fi 271 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 272 | # Running the downloader 273 | if [ "$MVNW_VERBOSE" = true ]; then 274 | echo " - Running MavenWrapperDownloader.java ..." 275 | fi 276 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 277 | fi 278 | fi 279 | fi 280 | fi 281 | ########################################################################################## 282 | # End of extension 283 | ########################################################################################## 284 | 285 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 286 | if [ "$MVNW_VERBOSE" = true ]; then 287 | echo $MAVEN_PROJECTBASEDIR 288 | fi 289 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 290 | 291 | # For Cygwin, switch paths to Windows format before running java 292 | if $cygwin; then 293 | [ -n "$M2_HOME" ] && 294 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 295 | [ -n "$JAVA_HOME" ] && 296 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 297 | [ -n "$CLASSPATH" ] && 298 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 299 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 300 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 301 | fi 302 | 303 | # Provide a "standardized" way to retrieve the CLI args that will 304 | # work with both Windows and non-Windows executions. 305 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 306 | export MAVEN_CMD_LINE_ARGS 307 | 308 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 309 | 310 | exec "$JAVACMD" \ 311 | $MAVEN_OPTS \ 312 | $MAVEN_DEBUG_OPTS \ 313 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 314 | "-Dmaven.home=${M2_HOME}" \ 315 | "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 316 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 317 | -------------------------------------------------------------------------------- /master/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* 50 | if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 124 | 125 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% ^ 162 | %JVM_CONFIG_MAVEN_PROPS% ^ 163 | %MAVEN_OPTS% ^ 164 | %MAVEN_DEBUG_OPTS% ^ 165 | -classpath %WRAPPER_JAR% ^ 166 | "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ 167 | %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 168 | if ERRORLEVEL 1 goto error 169 | goto end 170 | 171 | :error 172 | set ERROR_CODE=1 173 | 174 | :end 175 | @endlocal & set ERROR_CODE=%ERROR_CODE% 176 | 177 | if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost 178 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 179 | if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" 180 | if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" 181 | :skipRcPost 182 | 183 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 184 | if "%MAVEN_BATCH_PAUSE%"=="on" pause 185 | 186 | if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% 187 | 188 | cmd /C exit /B %ERROR_CODE% 189 | -------------------------------------------------------------------------------- /master/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.6.7 9 | 10 | 11 | xyz.ralxyz 12 | minisql_master 13 | 0.0.1-SNAPSHOT 14 | master 15 | master 16 | 17 | 17 18 | 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter 23 | 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-test 28 | test 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-web 33 | 34 | 35 | org.apache.curator 36 | curator-recipes 37 | 5.2.1 38 | 39 | 40 | org.apache.curator 41 | curator-test 42 | 5.2.1 43 | 44 | 45 | org.projectlombok 46 | lombok 47 | 48 | 49 | com.alibaba 50 | fastjson 51 | 2.0.2 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/Cluster.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.curator.framework.CuratorFramework; 5 | import org.apache.curator.framework.CuratorFrameworkFactory; 6 | import org.apache.curator.framework.recipes.cache.CuratorCache; 7 | import org.apache.curator.framework.recipes.cache.CuratorCacheListener; 8 | import org.apache.curator.retry.ExponentialBackoffRetry; 9 | import org.apache.curator.test.TestingCluster; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.stereotype.Service; 13 | 14 | import java.io.BufferedReader; 15 | import java.io.File; 16 | import java.io.IOException; 17 | import java.io.InputStreamReader; 18 | import java.util.ArrayList; 19 | import java.util.Arrays; 20 | import java.util.List; 21 | import java.util.concurrent.ConcurrentHashMap; 22 | import java.util.concurrent.ExecutorService; 23 | import java.util.concurrent.Executors; 24 | 25 | @Slf4j 26 | @Service 27 | public class Cluster { 28 | public TestingCluster testingCluster; 29 | 30 | public CuratorFramework client; 31 | 32 | public static ExecutorService threadPool = Executors.newFixedThreadPool(2); 33 | 34 | public ConcurrentHashMap> zkPathMap = new ConcurrentHashMap<>(); 35 | 36 | @Autowired 37 | Config config; 38 | 39 | @Bean 40 | private void init() throws Exception { 41 | this.testingCluster = new TestingCluster(3); 42 | this.testingCluster.start(); 43 | 44 | this.client = CuratorFrameworkFactory.builder() 45 | .connectString(this.testingCluster.getConnectString()) 46 | .retryPolicy(new ExponentialBackoffRetry(1000, 3)) 47 | .build(); 48 | this.client.start(); 49 | 50 | this.client.create().creatingParentsIfNeeded().forPath("/db"); 51 | 52 | creteCallback(); 53 | startRegions(); 54 | } 55 | 56 | private void creteCallback() { 57 | var cache = CuratorCache.build(this.client, "/db"); 58 | cache.start(); 59 | 60 | cache.listenable().addListener( 61 | (eventType, oldData, newData) -> { 62 | if (eventType == CuratorCacheListener.Type.NODE_DELETED) { 63 | final var name = Arrays.stream(oldData.getPath().split("/")).reduce((a, b) -> b).get(); 64 | var connectStringList = java.util.Arrays.stream(this.testingCluster.getConnectString().split(",")).toList(); 65 | zkPathMap.remove(name); 66 | final var i = config.zkPathName.indexOf(name); 67 | try { 68 | new ProcessBuilder().command( 69 | "./MiniSQL", 70 | config.zkPathName.get(i), 71 | connectStringList.get(i), 72 | config.regionPort.get(i) 73 | ).directory(new File(config.shellPath.get(i))).start(); 74 | } catch (IOException e) { 75 | throw new RuntimeException(e); 76 | } 77 | return; 78 | } 79 | log.info("create node: " + newData.getPath()); 80 | log.info("create type: " + eventType); 81 | log.info("create data: " + new String(newData.getData())); 82 | 83 | switch (eventType) { 84 | case NODE_CREATED, NODE_CHANGED -> { 85 | if (!newData.getPath().equals("/db")) { 86 | final var tableListRaw = new String(newData.getData()); 87 | zkPathMap.put( 88 | Arrays.stream(newData.getPath().split("/")).reduce((a, b) -> b).get(), 89 | tableListRaw.length() == 0 ? new ArrayList<>() : Arrays.stream(tableListRaw.split(",")).toList() 90 | ); 91 | } 92 | } 93 | } 94 | }, 95 | threadPool 96 | ); 97 | } 98 | 99 | private void startRegions() throws Exception { 100 | var connectStringList = java.util.Arrays.stream(this.testingCluster.getConnectString().split(",")).toList(); 101 | for (int i = 0; i < connectStringList.size(); i++) { 102 | var builder = new ProcessBuilder(); 103 | builder.command( 104 | "./MiniSQL", 105 | config.zkPathName.get(i), 106 | connectStringList.get(i), 107 | config.regionPort.get(i) 108 | ).directory(new File(config.shellPath.get(i))); 109 | 110 | log.info("shellPath: " + config.shellPath.get(i)); 111 | log.info("zkPathName: " + config.zkPathName.get(i)); 112 | log.info("zkUrl: " + connectStringList.get(i)); 113 | log.info("regionPort: " + config.regionPort.get(i)); 114 | 115 | log.info("start: " + builder.command().stream().reduce((a, b) -> a + " " + b).get()); 116 | 117 | var process = builder.start(); 118 | } 119 | }; 120 | } 121 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/Config.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master; 2 | 3 | import org.springframework.beans.factory.annotation.Value; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | import java.util.List; 7 | 8 | @Configuration 9 | public class Config { 10 | @Value("${minisql.zk_url}") 11 | public List zkUrl; 12 | 13 | @Value("${minisql.zk_path_name}") 14 | public List zkPathName; 15 | 16 | @Value("${minisql.region_port}") 17 | public List regionPort; 18 | 19 | @Value("${minisql.shell_path}") 20 | public List shellPath; 21 | 22 | @Value("${minisql.region_replica}") 23 | public int regionReplica; 24 | 25 | } 26 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/Controller.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master; 2 | 3 | import com.alibaba.fastjson2.JSONObject; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.http.HttpEntity; 7 | import org.springframework.http.HttpHeaders; 8 | import org.springframework.http.MediaType; 9 | import org.springframework.web.bind.annotation.GetMapping; 10 | import org.springframework.web.bind.annotation.PostMapping; 11 | import org.springframework.web.bind.annotation.RequestBody; 12 | import org.springframework.web.bind.annotation.RestController; 13 | import org.springframework.web.client.RestTemplate; 14 | import xyz.ralxyz.minisql_master.model.RegionInfo; 15 | import xyz.ralxyz.minisql_master.model.SqlResponse; 16 | import xyz.ralxyz.minisql_master.model.Statement; 17 | 18 | import java.net.SocketException; 19 | import java.util.*; 20 | 21 | @Slf4j 22 | @RestController 23 | public class Controller { 24 | 25 | Random rand = new Random(); 26 | 27 | @Autowired 28 | Cluster cluster; 29 | 30 | @Autowired 31 | Config config; 32 | 33 | @GetMapping("/cache") 34 | public List getCache() { 35 | final var regionInfoList = new ArrayList(); 36 | 37 | log.info(cluster.zkPathMap.toString()); 38 | 39 | cluster.zkPathMap.forEach((k, v) -> 40 | regionInfoList.add( 41 | new RegionInfo(v, getRegionUrl(k)) 42 | ) 43 | ); 44 | 45 | log.info(cluster.testingCluster.getConnectString()); 46 | log.info(regionInfoList.toString()); 47 | 48 | return regionInfoList; 49 | } 50 | 51 | @PostMapping("/statement") 52 | public SqlResponse postStatement(@RequestBody final Statement statement) throws Exception { 53 | final var regionList = new ArrayList(); 54 | var isCreate = false; 55 | var isDrop = false; 56 | var isExit = false; 57 | 58 | final var commandList = Arrays.stream(statement.command().toLowerCase().split(" +")).toList(); 59 | 60 | if (commandList.size() > 0 && commandList.get(0).startsWith("exit")) { 61 | isExit = true; 62 | } 63 | 64 | boolean finalIsExit = isExit; 65 | cluster.zkPathMap.forEach((k, v) -> { 66 | if (finalIsExit || v.contains(statement.tableName())) { 67 | regionList.add(k); 68 | } 69 | }); 70 | 71 | 72 | if (commandList.size() >= 2 && commandList.get(0).equals("create") && commandList.get(1).equals("table")) { 73 | isCreate = true; 74 | if (regionList.size() != 0) { 75 | return new SqlResponse(-1, "ERROR 200 from : table '" + statement.tableName() + "' exists"); 76 | } else { 77 | // rand regions, add them to regionUrlList\ 78 | while (regionList.size() < config.regionReplica) { 79 | final var randIdx = rand.nextInt(cluster.zkPathMap.size()); 80 | final var region = (String)cluster.zkPathMap.keySet().toArray()[randIdx]; 81 | if (!regionList.contains(region)) { 82 | regionList.add(region); 83 | } 84 | } 85 | } 86 | } else if (commandList.size() >= 2 && commandList.get(0).equals("drop") && commandList.get(1).equals("table")) { 87 | isDrop = true; 88 | } 89 | 90 | SqlResponse resp = null; 91 | 92 | log.info(regionList.toString()); 93 | 94 | for (final var region : regionList) { 95 | final var headers = new HttpHeaders(); 96 | headers.setContentType(MediaType.APPLICATION_JSON); 97 | final var cmdJsonObject = new JSONObject(); 98 | cmdJsonObject.put("command", statement.command()); 99 | 100 | log.info(cmdJsonObject.toString()); 101 | 102 | try { 103 | final var personResultAsJsonStr = new RestTemplate().postForObject( 104 | "http://" + getRegionUrl(region), 105 | new HttpEntity<>(cmdJsonObject.toString(), headers), 106 | SqlResponse.class 107 | ); 108 | resp = personResultAsJsonStr; 109 | } catch (Exception e) { 110 | resp = new SqlResponse(1, "bye!"); 111 | } 112 | 113 | // normal response (0) or exit (1) 114 | if (resp.code() < 0) { 115 | break; 116 | } 117 | 118 | var tableListRaw = new String(this.cluster.client.getData().forPath("/db/" + region)); 119 | if (isCreate) { 120 | if (tableListRaw.length() > 0) { 121 | tableListRaw += ","; 122 | } 123 | tableListRaw += statement.tableName(); 124 | } else if (isDrop) { 125 | var tableList = tableListRaw.split(","); 126 | // remove tableName from tableList 127 | tableList = Arrays.stream(tableList).filter(s -> !s.equals(statement.tableName())).toArray(String[]::new); 128 | tableListRaw = String.join(",", tableList); 129 | } 130 | 131 | this.cluster.client.setData().forPath("/db/" + region, tableListRaw.getBytes()); 132 | } 133 | 134 | if (resp == null) { 135 | resp = new SqlResponse(-1, "ERROR: invalid table name"); 136 | } 137 | 138 | return resp; 139 | } 140 | 141 | private String getRegionUrl(final String pathName) { 142 | return "localhost:" + config.regionPort.get(config.zkPathName.indexOf(pathName)); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/MiniSqlMasterApplication.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class MiniSqlMasterApplication { 8 | public static void main(String[] args) { 9 | SpringApplication.run(MiniSqlMasterApplication.class, args); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/model/RegionInfo.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master.model; 2 | 3 | import java.util.List; 4 | 5 | public record RegionInfo( 6 | List tables, 7 | String regionUrl 8 | ) {} 9 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/model/SqlResponse.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master.model; 2 | 3 | public record SqlResponse(Integer code, String msg) { } 4 | -------------------------------------------------------------------------------- /master/src/main/java/xyz/ralxyz/minisql_master/model/Statement.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master.model; 2 | 3 | public record Statement(String command, String tableName) { } 4 | -------------------------------------------------------------------------------- /master/src/test/java/xyz/ralxyz/minisql_master/MiniSqlMasterApplicationTests.java: -------------------------------------------------------------------------------- 1 | package xyz.ralxyz.minisql_master; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class MiniSqlMasterApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | --------------------------------------------------------------------------------