├── .gitignore ├── README.md ├── pom.xml ├── src ├── main │ └── java │ │ └── com │ │ └── northeastern │ │ └── edu │ │ └── simpledb │ │ ├── backend │ │ ├── Launcher.java │ │ ├── common │ │ │ ├── AbstractCache.java │ │ │ └── SubArray.java │ │ ├── dm │ │ │ ├── DataManger.java │ │ │ ├── DataMangerHandler.java │ │ │ ├── Recover.java │ │ │ ├── cache │ │ │ │ ├── PageCache.java │ │ │ │ └── PageCacheHandler.java │ │ │ ├── dataItem │ │ │ │ ├── AbstractDataItem.java │ │ │ │ └── DataItem.java │ │ │ ├── logger │ │ │ │ ├── AbstractLogger.java │ │ │ │ └── Logger.java │ │ │ ├── page │ │ │ │ ├── AbstractPage.java │ │ │ │ ├── L1Page.java │ │ │ │ ├── Page.java │ │ │ │ └── SecondaryPage.java │ │ │ └── pageIndex │ │ │ │ ├── PageIndex.java │ │ │ │ └── PageInfo.java │ │ ├── im │ │ │ ├── BPlusTree.java │ │ │ └── Node.java │ │ ├── parser │ │ │ ├── Parser.java │ │ │ ├── Tokenizer.java │ │ │ └── statement │ │ │ │ ├── Abort.java │ │ │ │ ├── Begin.java │ │ │ │ ├── Commit.java │ │ │ │ ├── Create.java │ │ │ │ ├── Delete.java │ │ │ │ ├── Drop.java │ │ │ │ ├── Insert.java │ │ │ │ ├── Select.java │ │ │ │ ├── Show.java │ │ │ │ ├── SingleExpression.java │ │ │ │ ├── Update.java │ │ │ │ └── Where.java │ │ ├── server │ │ │ ├── Executor.java │ │ │ └── Server.java │ │ ├── tbm │ │ │ ├── AbstractTableManager.java │ │ │ ├── BeginRes.java │ │ │ ├── Booter.java │ │ │ ├── Field.java │ │ │ ├── FieldCalRes.java │ │ │ ├── Table.java │ │ │ └── TableManager.java │ │ ├── tm │ │ │ ├── AbstractTransactionManager.java │ │ │ └── TransactionManager.java │ │ ├── utils │ │ │ ├── Panic.java │ │ │ ├── ParseStringRes.java │ │ │ ├── Parser.java │ │ │ ├── RandomUtil.java │ │ │ └── Types.java │ │ └── vm │ │ │ ├── Entry.java │ │ │ ├── LockTable.java │ │ │ ├── Transaction.java │ │ │ ├── VersionManager.java │ │ │ ├── VersionManagerHandler.java │ │ │ └── Visibility.java │ │ ├── client │ │ ├── Client.java │ │ ├── Launcher.java │ │ ├── RoundTripper.java │ │ └── Shell.java │ │ ├── common │ │ └── Error.java │ │ └── transport │ │ ├── Encoder.java │ │ ├── Package.java │ │ ├── Packager.java │ │ └── Transporter.java └── test │ └── java │ └── com │ └── northeastern │ └── edu │ └── simpledb │ ├── backend │ ├── common │ │ └── AbstractCacheTest.java │ ├── dm │ │ ├── DataManagerTest.java │ │ ├── RecoverTest.java │ │ └── logger │ │ │ └── LoggerTest.java │ ├── im │ │ └── BPlusTreeTest.java │ ├── pageIndex │ │ └── PageIndexTest.java │ ├── parser │ │ ├── ParserTest.java │ │ └── TokenizerTest.java │ ├── tm │ │ └── TransactionManagerTest.java │ └── vm │ │ ├── LockTableTest.java │ │ └── VersionManagerTest.java │ └── transport │ └── PackageTest.java ├── start-client.sh └── start-server.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # IDEA file 2 | .idea/* 3 | 4 | # Temporary serialized file 5 | *.ser 6 | 7 | # DB data 8 | *.bt 9 | *.db 10 | *.log 11 | *.xid 12 | 13 | # Compiled file 14 | target/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simple DB 2 | ______ 3 | - Data reliability and data recovery 4 | - Two-stage locking protocol (2PL) implements serializable scheduling and MVCC 5 | - Two transaction isolation levels (read committed and repeatable read) 6 | - Deadlock handling 7 | - Simple table, field management and SQL parsing 8 | - Socket-based server and client 9 | ## How to start 10 | ______ 11 | ### Pre-requirement 12 | - OpenJDK 11 13 | - Apache Maven 3.9.5 14 | ### Shell 15 | - step1 16 | ```bash 17 | chmod +x ./start-server.sh ./start-client.sh 18 | ``` 19 | - step2 20 | ```bash 21 | ./start-server.sh 22 | ./start-client.sh 23 | ``` 24 | ## Dependency graph 25 | ______ 26 | ```mermaid 27 | graph TD; 28 | IM -->|Dependency| DM; 29 | DM -->|Dependency| TM; 30 | TBM -->|Dependency| VM; 31 | TBM -->|Dependency| IM; 32 | VM -->|Dependency| TM; 33 | VM -->|Dependency| DM; 34 | ``` 35 | ## Transaction Manager 36 | ______ 37 | ### Transaction Manager 38 | - builder pattern 39 | - responsible for state of transaction 40 | - maintaining a file starting with a `xid` 41 | ## Data Manager 42 | ______ 43 | ### Abstract Cache 44 | - template pattern 45 | - reference counting 46 | ### Logger 47 | - iterator pattern 48 | - format 49 | - log file: `[Xchecksum][log1][log2][log3]...[logN][BadTail]` 50 | - log: `[Size][Checksum][Data]` 51 | - data 52 | - update: `[DataType: 1byte][Xid: 8bytes][Uid: 8bytes][OldRaw][LatestRaw]` 53 | - uid: `[PageNumber: 4bytes][Offset: 4bytes]` 54 | - insert: `[DataType: 1byte][Xid: 8bytes][PageNumber: 4bytes][Offset: 2bytes][Raw]` 55 | - process 56 | - redo: scan log and process redo logs which state is committed 57 | - undo: scan log and process undo logs which state is active reversely 58 | - logging 59 | 1. wrap data to log 60 | 2. write log to log file 61 | 3. update check sum 62 | 4. flush to disk 63 | ### Page 64 | - format 65 | - level1 page 66 | - when db start up, write random bytes from `OF_VC` to `OF_VC + LEN_VC - 1` byte 67 | - when db close, copy random bytes from `OF_VC + LEN_VC` to `OF_VC + 2 * LEN_VC - 1` byte 68 | - used to determine whether the database was shut down normally in the last time 69 | - secondary page 70 | - `[Offset: 2bytes][Raw]` 71 | - stored FSO (free space offset) 72 | - used to process raw data as secondary page when page cache need it in the process of initialization of DataManager 73 | ### Page Index 74 | - splitting the secondary page into four-ty slots 75 | > the page index caches the free space of each page, it is used to quickly find a page with suitable space when the upper module performs an insertion operation without checking the information of each page from the disk or cache. 76 | ### Data Item 77 | - format 78 | - data item: `[ValidFlag: 1byte][DataSize: 3bytes][Raw]` 79 | ### Data Manager 80 | - feature: providing data accessing service for the upper layer, eg: vm, im 81 | - method 82 | - insert() 83 | - read() 84 | - update() 85 | ## Index Manager 86 | ______ 87 | ### Index Manager 88 | - format 89 | - Node: `[NumberOfKeys][SiblingUid][Son0][Key0][Son1][Key1]...[SonN][KeyN]` 90 | - insert() 91 | 1. assume `BALANCE` is `3` 92 | ```mermaid 93 | graph TD 94 | A(root) --> B(4) 95 | A --> C(7) 96 | A --> D(10) 97 | A --> E(13) 98 | B --> F(1) 99 | B --> G(2) 100 | B --> H(3) 101 | C --> I(4) 102 | C --> J(5) 103 | C --> K(6) 104 | D --> L(7) 105 | D --> M(8) 106 | D --> N(9) 107 | E --> O(10) 108 | E --> P(11) 109 | E --> Q(12) 110 | A --> U(MAX) 111 | U --> R(13) 112 | U --> S(14) 113 | U --> T(15) 114 | U --> V(16) 115 | U --> W(17) 116 | ``` 117 | 2. after insert `18`, `MAX` will be split, `16` will be created for `13, 14, 15`, then `MAX` will be created for `16, 17, 18` 118 | ```mermaid 119 | graph TD 120 | A(root) --> B(4) 121 | A --> C(7) 122 | A --> D(10) 123 | A --> E(13) 124 | B --> F(1) 125 | B --> G(2) 126 | B --> H(3) 127 | C --> I(4) 128 | C --> J(5) 129 | C --> K(6) 130 | D --> L(7) 131 | D --> M(8) 132 | D --> N(9) 133 | E --> O(10) 134 | E --> P(11) 135 | E --> Q(12) 136 | A --> U(MAX) 137 | U --> R(13) 138 | U --> S(14) 139 | U --> T(15) 140 | U --> V(16) 141 | U --> W(17) 142 | U --> X(18) 143 | ``` 144 | ```mermaid 145 | graph TD 146 | A(root) --> B(4) 147 | A --> C(7) 148 | A --> D(10) 149 | A --> E(13) 150 | B --> F(1) 151 | B --> G(2) 152 | B --> H(3) 153 | C --> I(4) 154 | C --> J(5) 155 | C --> K(6) 156 | D --> L(7) 157 | D --> M(8) 158 | D --> N(9) 159 | E --> O(10) 160 | E --> P(11) 161 | E --> Q(12) 162 | A --> Y(16) 163 | Y --> R(13) 164 | Y --> S(14) 165 | Y --> T(15) 166 | U(MAX) --> V(16) 167 | U --> W(17) 168 | U --> X(18) 169 | ``` 170 | 3. `MAX` will be inserted into `root`, so the number of son of `root` exceed limitation 171 | 4. `root` will be split, then another one `MAX` created 172 | ```mermaid 173 | graph TD 174 | A(root) --> B(4) 175 | A --> C(7) 176 | A --> D(10) 177 | B --> F(1) 178 | B --> G(2) 179 | B --> H(3) 180 | C --> I(4) 181 | C --> J(5) 182 | C --> K(6) 183 | D --> L(7) 184 | D --> M(8) 185 | D --> N(9) 186 | E --> O(10) 187 | E --> P(11) 188 | E --> Q(12) 189 | Y --> R(13) 190 | Y --> S(14) 191 | Y --> T(15) 192 | U(MAX) --> V(16) 193 | U --> W(17) 194 | U --> X(18) 195 | Z(MAX) --> E(13) 196 | Z --> Y(16) 197 | Z --> U 198 | ``` 199 | 5. create a `newRoot` and replace old root with it 200 | ```mermaid 201 | graph TD 202 | A(13) --> B(4) 203 | A --> C(7) 204 | A --> D(10) 205 | B --> F(1) 206 | B --> G(2) 207 | B --> H(3) 208 | C --> I(4) 209 | C --> J(5) 210 | C --> K(6) 211 | D --> L(7) 212 | D --> M(8) 213 | D --> N(9) 214 | E --> O(10) 215 | E --> P(11) 216 | E --> Q(12) 217 | U --> R(13) 218 | U --> S(14) 219 | U --> T(15) 220 | W --> X(16) 221 | W --> Y(17) 222 | W --> Z(18) 223 | V(MAX) --> E(13) 224 | V --> U(16) 225 | V --> W(MAX) 226 | newRoot --> A 227 | newRoot --> V 228 | ``` 229 | - search() 230 | ## Version Manager 231 | ______ 232 | ### Version Manager 233 | - two-phase locking & multi-version concurrency control 234 | - deadlock detection by depth first search 235 | - format 236 | - entry: `[Xmin][Xmax][Data]` 237 | ## Table Manager 238 | ______ 239 | ### Parser 240 | - lexical parser and grammar parser 241 | - finite state machine 242 | ```mermaid 243 | graph TD 244 | INIT -->|Symbol| IN_SYMBOL 245 | INIT -->|QuoteChar| IN_QUOTE 246 | INIT -->|Alphabet/Digit| IN_TOKEN 247 | IN_SYMBOL -->|Any| INIT 248 | IN_QUOTE -->|Any| IN_QUOTE 249 | IN_QUOTE -->|QuoteChar| INIT 250 | IN_TOKEN -->|Alphabet/Digit| IN_TOKEN 251 | IN_TOKEN -->|Blank| INIT 252 | ``` 253 | ### Table Manager 254 | - format 255 | - field: `[FieldName][TypeName][IndexUid]` 256 | - `[StringLength][StringData]` 257 | - table: `[TableName][NextTable][Fiedl1Uid][Field2Uid]...[FieldNUid]` 258 | - safe/atomic update of files via temporary files 259 | > ref: https://mozillazg.com/2018/05/a-way-to-atomic-write-or-safe-update-a-file.html 260 | - when creating a new table, the header interpolation method is used, so the Booter file needs to be updated every time a table is created. 261 | ## Transport 262 | ______ 263 | ### C/S 264 | - A special binary format is used for communication between the client and the server. 265 | - In order to avoid problems caused by special characters, the data will be converted into a hexadecimal string (Hex String). 266 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 4.0.0 6 | 7 | com.northeastern.edu 8 | simple-db 9 | 1.0-SNAPSHOT 10 | jar 11 | 12 | simple-db Maven Webapp 13 | 14 | 15 | UTF-8 16 | 11 17 | 11 18 | 19 | 20 | 21 | 22 | com.google.guava 23 | guava 24 | 32.1.2-jre 25 | 26 | 27 | commons-codec 28 | commons-codec 29 | 1.14 30 | 31 | 32 | commons-cli 33 | commons-cli 34 | 1.5.0 35 | 36 | 37 | org.junit.jupiter 38 | junit-jupiter-api 39 | 5.9.1 40 | test 41 | 42 | 43 | org.mockito 44 | mockito-core 45 | 4.0.0 46 | test 47 | 48 | 49 | com.google.code.gson 50 | gson 51 | 2.8.9 52 | test 53 | 54 | 55 | 56 | 57 | simple-db 58 | 59 | 60 | 61 | maven-clean-plugin 62 | 3.1.0 63 | 64 | 65 | 66 | maven-resources-plugin 67 | 3.0.2 68 | 69 | 70 | maven-compiler-plugin 71 | 3.8.0 72 | 73 | 74 | maven-surefire-plugin 75 | 2.22.1 76 | 77 | 78 | maven-war-plugin 79 | 3.2.2 80 | 81 | 82 | maven-install-plugin 83 | 2.5.2 84 | 85 | 86 | maven-deploy-plugin 87 | 2.8.2 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/Launcher.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler; 5 | import com.northeastern.edu.simpledb.backend.server.Server; 6 | import com.northeastern.edu.simpledb.backend.tbm.TableManager; 7 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 8 | import com.northeastern.edu.simpledb.backend.utils.Panic; 9 | import com.northeastern.edu.simpledb.backend.vm.VersionManager; 10 | import com.northeastern.edu.simpledb.common.Error; 11 | import org.apache.commons.cli.CommandLine; 12 | import org.apache.commons.cli.DefaultParser; 13 | import org.apache.commons.cli.Options; 14 | import org.apache.commons.cli.ParseException; 15 | 16 | public class Launcher { 17 | 18 | public static final int port = 9999; 19 | 20 | public static final long DEFALUT_MEM = (1<<20)*64; 21 | 22 | public static final long KB = 1 << 10; 23 | public static final long MB = 1 << 20; 24 | public static final long GB = 1 << 30; 25 | 26 | public static void main(String[] args) throws ParseException { 27 | Options options = new Options(); 28 | options.addOption("open", true, "-open DBPath"); 29 | options.addOption("create", true, "-create DBPath"); 30 | options.addOption("mem", true, "-mem 64MB"); 31 | DefaultParser parser = new DefaultParser(); 32 | CommandLine cmd = parser.parse(options, args); 33 | 34 | if(cmd.hasOption("open")) { 35 | openDB(cmd.getOptionValue("open"), parseMem(cmd.getOptionValue("mem"))); 36 | return; 37 | } 38 | if(cmd.hasOption("create")) { 39 | createDB(cmd.getOptionValue("create")); 40 | return; 41 | } 42 | System.out.println("Usage: launcher (open|create) DBPath"); 43 | } 44 | 45 | private static void createDB(String path) { 46 | TransactionManager tm = TransactionManager.create(path); 47 | DataManger dm = DataMangerHandler.create(path, DEFALUT_MEM, tm); 48 | VersionManager vm = new VersionManager(tm, dm); 49 | TableManager.create(path, vm, dm); 50 | tm.close(); 51 | dm.close(); 52 | } 53 | 54 | private static void openDB(String path, long mem) { 55 | TransactionManager tm = TransactionManager.open(path); 56 | DataManger dm = DataMangerHandler.open(path, mem, tm); 57 | VersionManager vm = new VersionManager(tm, dm); 58 | TableManager tbm = TableManager.open(path, vm, dm); 59 | new Server(port, tbm).start(); 60 | } 61 | 62 | private static long parseMem(String memStr) { 63 | if(memStr == null || "".equals(memStr)) { 64 | return DEFALUT_MEM; 65 | } 66 | if(memStr.length() < 2) { 67 | Panic.panic(Error.InvalidMemException); 68 | } 69 | String unit = memStr.substring(memStr.length()-2); 70 | long memNum = Long.parseLong(memStr.substring(0, memStr.length()-2)); 71 | switch(unit) { 72 | case "KB": 73 | return memNum*KB; 74 | case "MB": 75 | return memNum*MB; 76 | case "GB": 77 | return memNum*GB; 78 | default: 79 | Panic.panic(Error.InvalidMemException); 80 | } 81 | return DEFALUT_MEM; 82 | } 83 | 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/common/AbstractCache.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.common; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Set; 6 | import java.util.concurrent.ConcurrentHashMap; 7 | import java.util.concurrent.locks.Condition; 8 | import java.util.concurrent.locks.Lock; 9 | import java.util.concurrent.locks.ReentrantLock; 10 | 11 | import com.northeastern.edu.simpledb.common.Error; 12 | 13 | /** 14 | * AbstractCache is a reference counting strategy for caching 15 | */ 16 | public abstract class AbstractCache { 17 | 18 | // the actual cache for data 19 | private Map cache; 20 | 21 | // a counter for the reference 22 | private Map references; 23 | 24 | // whether the resource is being obtained by other thread 25 | private Map getting; 26 | 27 | // the maximum number of data in cache 28 | private int maxResource; 29 | 30 | // the number of elements in the cache 31 | private int count = 0; 32 | 33 | private Lock lock; 34 | 35 | private Condition condition; 36 | 37 | public AbstractCache(int maxResource) { 38 | this.maxResource = maxResource; 39 | this.cache = new ConcurrentHashMap<>(); 40 | this.references = new ConcurrentHashMap<>(); 41 | this.getting = new ConcurrentHashMap<>(); 42 | this.lock = new ReentrantLock(); 43 | this.condition = lock.newCondition(); 44 | } 45 | 46 | // obtain data based on key 47 | protected T get(long key) throws Exception { 48 | 49 | lock.lock(); 50 | 51 | try { 52 | // the thread starts waiting while the data the thread wants is being loaded into the cache 53 | while (getting.containsKey(key)) { 54 | try { 55 | condition.await(); 56 | } catch (InterruptedException e) { 57 | throw new RuntimeException(e); 58 | } 59 | } 60 | 61 | if (cache.containsKey(key)) { 62 | T obj = cache.get(key); 63 | references.put(key, references.get(key) + 1); 64 | return obj; 65 | } 66 | 67 | if(maxResource > 0 && count == maxResource) { 68 | throw Error.CacheFullException; 69 | } 70 | 71 | count++; 72 | getting.put(key, true); 73 | } catch (Exception e) { 74 | throw e; 75 | } finally { 76 | lock.unlock(); 77 | } 78 | 79 | 80 | T obj = null; 81 | try { 82 | obj = getForCache(key); 83 | } catch(Exception e) { 84 | lock.lock(); 85 | try { 86 | count--; 87 | getting.remove(key); 88 | } finally { 89 | lock.unlock(); 90 | } 91 | throw e; 92 | } 93 | 94 | lock.lock(); 95 | try { 96 | getting.remove(key); 97 | cache.put(key, obj); 98 | references.put(key, 1); 99 | // after data is loaded into the cache, notify all threads is sleeping before 100 | condition.signalAll(); 101 | } finally { 102 | lock.unlock(); 103 | } 104 | 105 | return obj; 106 | } 107 | 108 | // forcefully release a cache based on key 109 | protected void release(long key) { 110 | lock.lock(); 111 | try { 112 | int ref = references.get(key) - 1; 113 | if (ref < 0) return ; 114 | if(ref == 0) { 115 | T obj = cache.get(key); 116 | releaseForCache(obj); 117 | references.remove(key); 118 | cache.remove(key); 119 | count--; 120 | } else { 121 | references.put(key, ref); 122 | } 123 | } finally { 124 | lock.unlock(); 125 | } 126 | } 127 | 128 | // turn off caching and write back all resources 129 | protected void close() { 130 | lock.lock(); 131 | try { 132 | Set keys = cache.keySet(); 133 | for (long key : keys) { 134 | T obj = cache.get(key); 135 | releaseForCache(obj); 136 | references.remove(key); 137 | cache.remove(key); 138 | } 139 | } finally { 140 | lock.unlock(); 141 | } 142 | } 143 | 144 | // default behavior to perform if cache misses 145 | protected abstract T getForCache(long key) throws Exception; 146 | 147 | // default write back behavior when cache eviction 148 | protected abstract void releaseForCache(T obj); 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/common/SubArray.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.common; 2 | 3 | public class SubArray { 4 | public byte[] raw; 5 | public int start; 6 | public int end; 7 | 8 | public SubArray(byte[] raw, int start, int end) { 9 | this.raw = raw; 10 | this.start = start; 11 | this.end = end; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/DataManger.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache; 4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger; 7 | import com.northeastern.edu.simpledb.backend.dm.page.L1Page; 8 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 9 | import com.northeastern.edu.simpledb.backend.dm.page.SecondaryPage; 10 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageIndex; 11 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageInfo; 12 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 13 | import com.northeastern.edu.simpledb.backend.utils.Panic; 14 | import com.northeastern.edu.simpledb.backend.utils.Types; 15 | import com.northeastern.edu.simpledb.common.Error; 16 | 17 | public class DataManger extends AbstractCache implements DataMangerHandler { 18 | 19 | TransactionManager tm; 20 | PageCache pageCache; 21 | Logger logger; 22 | PageIndex pageIndex; 23 | 24 | Page firstPage; 25 | 26 | public DataManger(PageCache pageCache, Logger logger, TransactionManager tm) { 27 | super(0); 28 | this.pageCache = pageCache; 29 | this.logger = logger; 30 | this.tm = tm; 31 | this.pageIndex = new PageIndex(); 32 | } 33 | 34 | void initL1Page() { 35 | int pageNumber = pageCache.newPage(L1Page.initRaw()); 36 | assert pageNumber == 1; 37 | try { 38 | firstPage = pageCache.getPage(pageNumber); 39 | } catch (Exception e) { 40 | Panic.panic(e); 41 | } 42 | pageCache.flushPage(firstPage); 43 | } 44 | 45 | boolean loadCheckFirstPage() { 46 | try { 47 | firstPage = pageCache.getPage(1); 48 | } catch (Exception e) { 49 | Panic.panic(e); 50 | } 51 | return L1Page.checkVc(firstPage); 52 | } 53 | 54 | // initialize page index after DataManger#open 55 | void fillPageIndex() { 56 | int pageNumber = pageCache.getPageNumber(); 57 | // iterate over all page number, then build page index 58 | for (int i = 2; i <= pageNumber; i++) { 59 | Page page = null; 60 | try { 61 | page = pageCache.getPage(i); 62 | } catch (Exception e) { 63 | Panic.panic(e); 64 | } 65 | pageIndex.add(pageNumber, SecondaryPage.getFreeSpace(page)); 66 | page.release(); 67 | } 68 | } 69 | 70 | // generate log for every transaction referring to `xid` 71 | public void logDataItem(long xid, DataItem dataItem) { 72 | byte[] log = Recover.updateLog(xid, dataItem); 73 | logger.log(log); 74 | } 75 | 76 | // build key from uid, then get data item by key 77 | @Override 78 | protected DataItem getForCache(long uid) throws Exception { 79 | short offset = (short) (uid & ((1L << 16) - 1)); 80 | uid >>>= 32; 81 | int pageNumber = (int) (uid & ((1L << 32) - 1)); 82 | Page page = pageCache.getPage(pageNumber); 83 | return DataItem.parseDataItem(page, offset, this); 84 | } 85 | 86 | @Override 87 | protected void releaseForCache(DataItem dataItem) { 88 | dataItem.page().release(); 89 | } 90 | 91 | // read a data by uid 92 | @Override 93 | public DataItem read(long uid) throws Exception { 94 | DataItem dataItem = super.get(uid); 95 | if (!dataItem.isValid()) { 96 | dataItem.release(); 97 | return null; 98 | } 99 | return dataItem; 100 | } 101 | 102 | // insert a data through DataManger 103 | @Override 104 | public long insert(long xid, byte[] data) throws Exception { 105 | byte[] raw = DataItem.wrapDataItemRaw(data); 106 | if (raw.length > SecondaryPage.MAX_FREE_SPACE) throw Error.DataTooLargeException; 107 | 108 | // TODO: need to refactor 109 | // step1 find page index 110 | PageInfo pageInfo = null; 111 | for (int i = 0; i < 5; i++) { 112 | pageInfo = pageIndex.select(raw.length); 113 | if (pageInfo != null) { 114 | break; 115 | } else { 116 | // if no page has enough free space, create a new secondary page 117 | int newPageNumber = pageCache.newPage(SecondaryPage.initRaw()); 118 | pageIndex.add(newPageNumber, SecondaryPage.MAX_FREE_SPACE); 119 | } 120 | } 121 | 122 | if (pageInfo == null) throw Error.DataBaseBusyException; 123 | 124 | // step2 write to log file 125 | Page page = null; 126 | int freeSpace = 0; 127 | 128 | try { 129 | page = pageCache.getPage(pageInfo.pageNumber); 130 | byte[] log = Recover.insertLog(xid, page, raw); 131 | logger.log(log); 132 | 133 | // step3 write to page, return offset can be used as uid for cache of data item 134 | short offset = SecondaryPage.insert(page, raw); 135 | page.release(); 136 | return Types.addressToUid(pageInfo.pageNumber, offset); 137 | } finally { 138 | // step4 put back page index 139 | if (page != null) { 140 | pageIndex.add(pageInfo.pageNumber, SecondaryPage.getFreeSpace(page)); 141 | } else { 142 | pageIndex.add(pageInfo.pageNumber, freeSpace); 143 | } 144 | } 145 | 146 | } 147 | 148 | // close cache and logger 149 | @Override 150 | public void close() { 151 | super.close(); 152 | logger.close(); 153 | 154 | L1Page.setVcClose(firstPage); 155 | firstPage.release(); 156 | pageCache.close(); 157 | } 158 | 159 | public void releaseDataItem(DataItem dataItem) { 160 | super.release(dataItem.getUid()); 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/DataMangerHandler.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCacheHandler; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger; 7 | import com.northeastern.edu.simpledb.backend.dm.page.L1Page; 8 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 9 | 10 | public interface DataMangerHandler { 11 | 12 | DataItem read(long uid) throws Exception; 13 | long insert(long xid, byte[] data) throws Exception; 14 | void close(); 15 | 16 | /** 17 | * create page cache, logger, and data manager 18 | * initialize the L1 page 19 | */ 20 | static DataManger create(String path, long memory, TransactionManager tm) { 21 | PageCache pageCache = PageCacheHandler.create(path, memory); 22 | Logger logger = Logger.create(path); 23 | 24 | DataManger dataManger = new DataManger(pageCache, logger, tm); 25 | dataManger.initL1Page(); 26 | return dataManger; 27 | } 28 | 29 | static DataManger open(String path, long memory, TransactionManager tm) { 30 | PageCache pageCache = PageCacheHandler.open(path, memory); 31 | Logger logger = Logger.open(path); 32 | 33 | DataManger dataManger = new DataManger(pageCache, logger, tm); 34 | if (!dataManger.loadCheckFirstPage()) { // check failed 35 | Recover.recover(tm, logger, pageCache); // recover based on log file 36 | } 37 | dataManger.fillPageIndex(); // initialize page index 38 | L1Page.setVcOpen(dataManger.firstPage); // set flag for valid check 39 | dataManger.pageCache.flushPage(dataManger.firstPage); // flush to disk immediately 40 | return dataManger; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/Recover.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.backend.common.SubArray; 5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 6 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 7 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger; 8 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 9 | import com.northeastern.edu.simpledb.backend.dm.page.SecondaryPage; 10 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 11 | import com.northeastern.edu.simpledb.backend.utils.Panic; 12 | import com.northeastern.edu.simpledb.backend.utils.Parser; 13 | 14 | import java.util.*; 15 | 16 | public class Recover { 17 | 18 | /** 19 | * insert log: [size][checksum][log type][xid][page number][offset][raw] 20 | * update log: [size][checksum][log type][xid][uid][old data][latest data] 21 | */ 22 | 23 | private static final byte LOG_TYPE_INSERT = 0; 24 | private static final byte LOG_TYPE_UPDATE = 1; 25 | 26 | private static final int OF_TYPE = 0; 27 | private static final int OF_XID = OF_TYPE + 1; 28 | 29 | private static final int OF_UPDATE_UID = OF_XID + 8; 30 | private static final int OF_UPDATE_RAW = OF_UPDATE_UID + 8; 31 | 32 | private static final int OF_INSERT_PAGE_NUMBER = OF_XID + 8; 33 | private static final int OF_INSERT_OFFSET = OF_INSERT_PAGE_NUMBER + 4; 34 | private static final int OF_INSERT_RAW = OF_INSERT_OFFSET + 2; 35 | 36 | public static byte[] insertLog(long xid, Page page, byte[] raw) { 37 | // [DataType: 1byte][Xid: 8bytes][PageNumber: 4bytes][Offset: 2bytes][Raw] 38 | byte[] logTypeRaw = {LOG_TYPE_INSERT}; 39 | byte[] xidRaw = Parser.long2Byte(xid); 40 | byte[] pageNumberRaw = Parser.int2Byte(page.getPageNumber()); 41 | byte[] offsetRaw = Parser.short2Byte(SecondaryPage.getFSO(page)); 42 | return Bytes.concat(logTypeRaw, xidRaw, pageNumberRaw, offsetRaw, raw); 43 | } 44 | 45 | static class InsertLogInfo { 46 | long xid; 47 | int pageNumber; 48 | short offset; 49 | byte[] raw; 50 | } 51 | 52 | static class UpdateLogInfo { 53 | long xid; 54 | int pageNumber; 55 | short offset; 56 | byte[] oldRaw; 57 | byte[] newRaw; 58 | } 59 | 60 | public static void recover(TransactionManager tm, Logger logger, PageCache pageCache) { 61 | System.out.println("Recovering"); 62 | 63 | logger.rewind(); 64 | int maxPage = 0; 65 | byte[] log; 66 | while ((log = logger.next()) != null) { 67 | int pageNumber; 68 | if (isInsertLog(log)) { 69 | InsertLogInfo li = parseInsertLog(log); 70 | pageNumber = li.pageNumber; 71 | } else { 72 | UpdateLogInfo xi = parseUpdateLog(log); 73 | pageNumber = xi.pageNumber; 74 | } 75 | if (pageNumber > maxPage) { 76 | maxPage = pageNumber; 77 | } 78 | } 79 | 80 | if (maxPage == 0) maxPage = 1; 81 | pageCache.truncateByPageNumber(maxPage); 82 | System.out.println("Truncate to " + maxPage + " pages."); 83 | 84 | redoTransactions(tm, logger, pageCache); 85 | System.out.println("Redo Transactions Over."); 86 | 87 | undoTransactions(tm, logger, pageCache); 88 | System.out.println("Undo Transactions Over."); 89 | 90 | System.out.println("Recovery Over."); 91 | 92 | } 93 | 94 | // redo transactions based on log file 95 | private static void redoTransactions(TransactionManager tm, Logger logger, PageCache pageCache) { 96 | logger.rewind(); 97 | byte[] log; 98 | while ((log = logger.next()) != null) { 99 | if (isInsertLog(log)) { 100 | InsertLogInfo li = parseInsertLog(log); 101 | if (!tm.isActive(li.xid)) { 102 | doInsertLog(pageCache, log, REDO); 103 | } 104 | } else { 105 | UpdateLogInfo xi = parseUpdateLog(log); 106 | if (!tm.isActive(xi.xid)) { 107 | doUpdateLog(pageCache, log, REDO); 108 | } 109 | } 110 | } 111 | } 112 | 113 | // [log type][xid][uid][old data][latest data] 114 | private static UpdateLogInfo parseUpdateLog(byte[] log) { 115 | UpdateLogInfo updateLogInfo = new UpdateLogInfo(); 116 | updateLogInfo.xid = Parser.parseLong(Arrays.copyOfRange(log, OF_XID, OF_UPDATE_UID)); 117 | // update id: [page number][offset] 118 | long uid = Parser.parseLong(Arrays.copyOfRange(log, OF_UPDATE_UID, OF_UPDATE_RAW)); 119 | updateLogInfo.offset = (short) (uid & ((1L << 16) - 1)); 120 | uid >>>= 16; 121 | updateLogInfo.pageNumber = (int) (uid & (1L << 32) - 1); 122 | int length = (log.length - OF_INSERT_RAW) / 2; 123 | updateLogInfo.oldRaw = Arrays.copyOfRange(log, OF_UPDATE_RAW, OF_UPDATE_RAW + length); 124 | updateLogInfo.newRaw = Arrays.copyOfRange(log, OF_UPDATE_RAW + length, OF_UPDATE_RAW + 2 * length); 125 | return updateLogInfo; 126 | } 127 | 128 | // [log type][xid][page number][offset][raw] 129 | private static InsertLogInfo parseInsertLog(byte[] log) { 130 | InsertLogInfo insertLogInfo = new InsertLogInfo(); 131 | insertLogInfo.xid = Parser.parseLong(Arrays.copyOfRange(log, OF_XID, OF_INSERT_PAGE_NUMBER)); 132 | insertLogInfo.pageNumber = Parser.parseInt(Arrays.copyOfRange(log, OF_INSERT_PAGE_NUMBER, OF_INSERT_OFFSET)); 133 | insertLogInfo.offset = Parser.parseShort(Arrays.copyOfRange(log, OF_INSERT_OFFSET, OF_INSERT_RAW)); 134 | insertLogInfo.raw = Arrays.copyOfRange(log, OF_INSERT_RAW, log.length); 135 | return insertLogInfo; 136 | } 137 | 138 | // determine type of log based on the first byte 139 | private static boolean isInsertLog(byte[] log) { 140 | return log[0] == LOG_TYPE_INSERT; 141 | } 142 | 143 | // update transactions based on log file 144 | private static void undoTransactions(TransactionManager tm, Logger logger, PageCache pageCache) { 145 | HashMap> logCache = new HashMap(); 146 | logger.rewind(); 147 | byte[] log; 148 | while ((log = logger.next()) != null) { 149 | if (isInsertLog(log)) { 150 | InsertLogInfo li = parseInsertLog(log); 151 | if (tm.isActive(li.xid)) { 152 | if (!logCache.containsKey(li.xid)) logCache.put(li.xid, new ArrayList<>()); 153 | logCache.get(li.xid).add(log); 154 | } 155 | } else { 156 | UpdateLogInfo xi = parseUpdateLog(log); 157 | if (tm.isActive(xi.xid)) { 158 | if (!logCache.containsKey(xi.xid)) logCache.put(xi.xid, new ArrayList<>()); 159 | logCache.get(xi.xid).add(log); 160 | } 161 | } 162 | } 163 | 164 | // processing undo logs which state is active reversely 165 | for (Map.Entry> entry : logCache.entrySet()) { 166 | List logs = entry.getValue(); 167 | for (int i = logs.size() - 1; i >= 0; i--) { 168 | byte[] tmp = logs.get(i); 169 | if (isInsertLog(tmp)) doInsertLog(pageCache, tmp, UNDO); 170 | else doUpdateLog(pageCache, tmp, UNDO); 171 | } 172 | tm.abort(entry.getKey()); 173 | } 174 | } 175 | 176 | 177 | private final static int REDO = 0; 178 | private final static int UNDO = 0; 179 | 180 | private static void doUpdateLog(PageCache pageCache, byte[] log, int flag) { 181 | // step1 determine action based on flag 182 | int pageNumber; 183 | short offset; 184 | byte[] raw; 185 | 186 | UpdateLogInfo updateLogInfo = parseUpdateLog(log); 187 | pageNumber = updateLogInfo.pageNumber; 188 | offset = updateLogInfo.offset; 189 | 190 | if (flag == REDO) raw = updateLogInfo.newRaw; // redo update 191 | else raw = updateLogInfo.oldRaw; // undo update 192 | 193 | // step2 get page by page number 194 | Page page = null; 195 | try { 196 | page = pageCache.getPage(pageNumber); 197 | } catch (Exception e) { 198 | Panic.panic(e); 199 | } 200 | 201 | // step3 update page with the help of `recoverUpdate()` from secondary page 202 | assert page != null; 203 | try { 204 | SecondaryPage.recoverUpdate(page, raw, offset); 205 | } finally { 206 | page.release(); 207 | } 208 | } 209 | 210 | private static void doInsertLog(PageCache pageCache, byte[] log, int flag) { 211 | // step1 parsing insert log 212 | InsertLogInfo insertLogInfo = parseInsertLog(log); 213 | int pageNumber = insertLogInfo.pageNumber; 214 | 215 | // step2 determine action based on flag 216 | if (flag == REDO) DataItem.setDataItemRawInvalid(insertLogInfo.raw); 217 | 218 | // step3 get page by page number 219 | Page page = null; 220 | try { 221 | page = pageCache.getPage(pageNumber); 222 | } catch (Exception e) { 223 | Panic.panic(e); 224 | } 225 | 226 | // step4 update page with the help of `recoverInsert()` from secondary page 227 | assert page != null; 228 | try { 229 | SecondaryPage.recoverInsert(page, insertLogInfo.raw, insertLogInfo.offset); 230 | } finally { 231 | page.release(); 232 | } 233 | } 234 | 235 | // when DataManger log into log file, data item should be wrapped in advance 236 | public static byte[] updateLog(long xid, DataItem dataItem) { 237 | byte[] logTypeRaw = {LOG_TYPE_UPDATE}; 238 | byte[] xidRaw = Parser.long2Byte(xid); 239 | byte[] uidRaw = Parser.long2Byte(dataItem.getUid()); 240 | byte[] oldRaw = dataItem.getOldRaw(); 241 | SubArray raw = dataItem.getRaw(); 242 | byte[] newRaw = Arrays.copyOfRange(raw.raw, raw.start, raw.end); 243 | return Bytes.concat(logTypeRaw, xidRaw, uidRaw, oldRaw, newRaw); 244 | } 245 | 246 | } 247 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/cache/PageCache.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.cache; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache; 4 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 5 | import com.northeastern.edu.simpledb.backend.utils.Panic; 6 | import com.northeastern.edu.simpledb.common.Error; 7 | 8 | import java.io.IOException; 9 | import java.io.RandomAccessFile; 10 | import java.nio.ByteBuffer; 11 | import java.nio.channels.FileChannel; 12 | import java.util.concurrent.atomic.AtomicInteger; 13 | import java.util.concurrent.locks.Lock; 14 | import java.util.concurrent.locks.ReentrantLock; 15 | 16 | public class PageCache extends AbstractCache implements PageCacheHandler { 17 | 18 | private static final int MEM_MIN_LIM = 10; 19 | 20 | public static final String DB_SUFFIX = ".db"; 21 | 22 | private RandomAccessFile file; 23 | 24 | private FileChannel fc; 25 | 26 | private Lock fileLock; 27 | 28 | private AtomicInteger pageNumbers; 29 | 30 | public PageCache(RandomAccessFile file, FileChannel fileChannel, int maxResource) { 31 | super(maxResource); 32 | if (maxResource < MEM_MIN_LIM) Panic.panic(Error.MemTooSmallException); 33 | long length = 0; 34 | try { 35 | length = file.length(); 36 | } catch (Exception e) { 37 | Panic.panic(e); 38 | } 39 | this.file = file; 40 | this.fc = fileChannel; 41 | this.fileLock = new ReentrantLock(); 42 | this.pageNumbers = new AtomicInteger((int) length / PAGE_SIZE); 43 | } 44 | 45 | // turn off caching and write back all resources 46 | @Override 47 | public void close() { 48 | super.close(); 49 | try { 50 | fc.close(); 51 | file.close(); 52 | } catch (IOException e) { 53 | Panic.panic(e); 54 | } 55 | } 56 | 57 | /** 58 | * read page from database file based on pageNumber, and 59 | * encapsulate it as Page 60 | */ 61 | @Override 62 | protected Page getForCache(long key) { 63 | int pageNumber = (int) key; 64 | long offset = pageOffset(pageNumber); 65 | 66 | ByteBuffer buf = ByteBuffer.allocate(PAGE_SIZE); 67 | fileLock.lock(); 68 | try { 69 | fc.position(offset); 70 | fc.read(buf); 71 | } catch (IOException e) { 72 | Panic.panic(e); 73 | } finally { 74 | fileLock.unlock(); 75 | } 76 | return new Page(pageNumber, buf.array(), this); 77 | } 78 | 79 | // calculate offset from the beginning based on page number 80 | static long pageOffset(int pageNumber) { 81 | return (long) (pageNumber - 1) * PAGE_SIZE; 82 | } 83 | 84 | // release cache, and write back to disk 85 | @Override 86 | protected void releaseForCache(Page page) { 87 | if (page.isDirty()) { 88 | flushPage(page); 89 | page.setDirty(false); 90 | } 91 | } 92 | 93 | // create a new page based on data, then flush to disk 94 | @Override 95 | public int newPage(byte[] initData) { 96 | int pageNumber = pageNumbers.incrementAndGet(); 97 | Page page = new Page(pageNumber, initData, null); 98 | flush(page); 99 | return pageNumber; 100 | } 101 | 102 | // get page from cache by page number 103 | @Override 104 | public Page getPage(int pageNumber) throws Exception { 105 | return get(pageNumber); 106 | } 107 | 108 | // release page from cache 109 | @Override 110 | public void release(Page page) { 111 | release(page.getPageNumber()); 112 | } 113 | 114 | // truncate file based on max page number 115 | @Override 116 | public void truncateByPageNumber(int maxPageNumber) { 117 | long size = pageOffset(maxPageNumber + 1); 118 | try { 119 | file.setLength(size); 120 | } catch (IOException e) { 121 | Panic.panic(e); 122 | } 123 | pageNumbers.set(maxPageNumber); 124 | } 125 | 126 | @Override 127 | public int getPageNumber() { 128 | return pageNumbers.intValue(); 129 | } 130 | 131 | @Override 132 | public void flushPage(Page page) { 133 | flush(page); 134 | } 135 | 136 | private void flush(Page page) { 137 | int pageNumber = page.getPageNumber(); 138 | long offset = pageOffset(pageNumber); 139 | 140 | fileLock.lock(); 141 | try { 142 | ByteBuffer buf = ByteBuffer.wrap(page.getData()); 143 | fc.position(offset); 144 | fc.write(buf); 145 | } catch (IOException e) { 146 | Panic.panic(e); 147 | } finally { 148 | fileLock.unlock(); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/cache/PageCacheHandler.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.cache; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 4 | import com.northeastern.edu.simpledb.backend.utils.Panic; 5 | import com.northeastern.edu.simpledb.common.Error; 6 | 7 | import java.io.File; 8 | import java.io.FileNotFoundException; 9 | import java.io.RandomAccessFile; 10 | import java.nio.channels.FileChannel; 11 | 12 | public interface PageCacheHandler { 13 | 14 | int PAGE_SIZE = 1 << 13; 15 | 16 | int newPage(byte[] initData); 17 | Page getPage(int pageNumber) throws Exception; 18 | void close(); 19 | void release(Page page); 20 | 21 | void truncateByPageNumber(int maxPageNumber); 22 | int getPageNumber(); 23 | void flushPage(Page pg); 24 | 25 | static PageCache create(String path, long memory) { 26 | File f = new File(path + PageCache.DB_SUFFIX); 27 | try { 28 | if (!f.createNewFile()) { 29 | Panic.panic(Error.FileExistsException); 30 | } 31 | } catch (Exception e) { 32 | Panic.panic(Error.FileCannotRWException); 33 | } 34 | 35 | FileChannel fc = null; 36 | RandomAccessFile raf = null; 37 | try { 38 | raf = new RandomAccessFile(f, "rw"); 39 | fc = raf.getChannel(); 40 | } catch (FileNotFoundException e) { 41 | Panic.panic(e); 42 | } 43 | return new PageCache(raf, fc, (int) memory / PAGE_SIZE); 44 | } 45 | 46 | static PageCache open(String path, long memory) { 47 | File f = new File(path + PageCache.DB_SUFFIX); 48 | if (!f.exists()) Panic.panic(Error.FileNotExistsException); 49 | if (!f.canRead() || !f.canWrite()) Panic.panic(Error.FileCannotRWException); 50 | 51 | FileChannel fc = null; 52 | RandomAccessFile raf = null; 53 | try { 54 | raf = new RandomAccessFile(f, "rw"); 55 | fc = raf.getChannel(); 56 | } catch (FileNotFoundException e) { 57 | Panic.panic(e); 58 | } 59 | return new PageCache(raf, fc, (int) memory / PAGE_SIZE); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/dataItem/AbstractDataItem.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.dataItem; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.backend.common.SubArray; 5 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 6 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler; 7 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 8 | import com.northeastern.edu.simpledb.backend.utils.Parser; 9 | import com.northeastern.edu.simpledb.backend.utils.Types; 10 | 11 | import java.util.Arrays; 12 | 13 | /** 14 | * AbstractDataItem 15 | * encapsulate data as DataItem and provide abstract objects 16 | * to the upper layer through DataManager 17 | */ 18 | public abstract class AbstractDataItem { 19 | 20 | abstract SubArray data(); 21 | 22 | abstract void before(); 23 | abstract void unBefore(); 24 | abstract void after(long xid); 25 | abstract void release(); 26 | 27 | abstract void lock(); 28 | abstract void unlock(); 29 | abstract void rLock(); 30 | abstract void rUnLock(); 31 | 32 | abstract Page page(); 33 | abstract long getUid(); 34 | abstract byte[] getOldRaw(); 35 | abstract SubArray getRaw(); 36 | 37 | // wrapping raw to DataItem 38 | public static byte[] wrapDataItemRaw(byte[] raw) { 39 | byte[] valid = new byte[1]; 40 | byte[] size = Parser.short2Byte((short) raw.length); 41 | return Bytes.concat(valid, size, raw); 42 | } 43 | 44 | // parsing raw to DataItem starting from the offset of page 45 | public static DataItem parseDataItem(Page page, short offset, DataManger dm) { 46 | byte[] raw = page.getData(); 47 | short dataSize = Parser.parseShort(Arrays.copyOfRange(raw, offset + DataItem.OF_SIZE, offset + DataItem.OF_DATA)); 48 | short length = (short) (dataSize + DataItem.OF_DATA); // total length of data item 49 | long uid = Types.addressToUid(page.getPageNumber(), offset); 50 | return new DataItem(new SubArray(raw, offset, offset + length), new byte[length], page, uid, dm); 51 | } 52 | 53 | public static void setDataItemRawInvalid(byte[] raw) { 54 | raw[DataItem.OF_VALID] = 0b1; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/dataItem/DataItem.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.dataItem; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.SubArray; 4 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 5 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 6 | 7 | import java.util.concurrent.locks.Lock; 8 | import java.util.concurrent.locks.ReadWriteLock; 9 | import java.util.concurrent.locks.ReentrantReadWriteLock; 10 | 11 | public class DataItem extends AbstractDataItem { 12 | 13 | static final int OF_VALID = 0; 14 | static final int OF_SIZE = 1; 15 | static final int OF_DATA = 3; 16 | 17 | static final byte VALID_STATE = 0b0; 18 | 19 | private SubArray raw; 20 | private byte[] oldRaw; 21 | private Lock rLock; 22 | private Lock wLock; 23 | private DataManger dm; 24 | private long uid; 25 | private Page page; 26 | 27 | public DataItem(SubArray raw, byte[] oldRaw, Page page, long uid, DataManger dm) { 28 | this.raw = raw; 29 | this.oldRaw = oldRaw; 30 | ReadWriteLock lock = new ReentrantReadWriteLock(); 31 | rLock = lock.readLock(); 32 | wLock = lock.writeLock(); 33 | this.dm = dm; 34 | this.uid = uid; 35 | this.page = page; 36 | } 37 | 38 | public boolean isValid() { 39 | return raw.raw[raw.start + OF_VALID] == VALID_STATE; 40 | } 41 | 42 | /** 43 | * The array returned by this method is data shared. Since 44 | * the array copy in Java will be allocated to a new address, 45 | * SubArray is used for data sharing between threads. 46 | */ 47 | @Override 48 | public SubArray data() { 49 | return new SubArray(raw.raw,raw.start + OF_DATA, raw.end); 50 | } 51 | 52 | /** 53 | * When the upper module tries to modify DataItem, it needs 54 | * to follow a certain process: before modification, the before() 55 | * method needs to be called. When you want to undo the modification, 56 | * the unBefore() method is called. After the modification is 57 | * completed, the after() method is called. The entire process 58 | * is mainly to save the previous phase data and log it in time. 59 | * DM will ensure that modifications to DataItem are atomic. 60 | */ 61 | 62 | // save the previous phase data 63 | @Override 64 | public void before() { 65 | wLock.lock(); 66 | page.setDirty(true); 67 | System.arraycopy(raw.raw, raw.start, oldRaw, 0, oldRaw.length); // latestRaw becomes oldRaw 68 | } 69 | 70 | // undo the previous action 71 | @Override 72 | public void unBefore() { 73 | wLock.lock(); 74 | page.setDirty(true); 75 | System.arraycopy(oldRaw, 0, raw.raw, raw.start, oldRaw.length); // oldRaw overwrite latestRaw 76 | } 77 | 78 | // log in time 79 | @Override 80 | public void after(long xid) { 81 | dm.logDataItem(xid, this); 82 | wLock.unlock(); 83 | } 84 | 85 | // after using data item, release it from DataManager 86 | @Override 87 | public void release() { 88 | dm.releaseDataItem(this); 89 | } 90 | 91 | @Override 92 | public void lock() { 93 | wLock.lock(); 94 | } 95 | 96 | @Override 97 | public void unlock() { 98 | wLock.unlock(); 99 | } 100 | 101 | @Override 102 | public void rLock() { 103 | rLock.lock(); 104 | } 105 | 106 | @Override 107 | public void rUnLock() { 108 | rLock.unlock(); 109 | } 110 | 111 | @Override 112 | public Page page() { 113 | return page; 114 | } 115 | 116 | @Override 117 | public long getUid() { 118 | return uid; 119 | } 120 | 121 | @Override 122 | public byte[] getOldRaw() { 123 | return oldRaw; 124 | } 125 | 126 | @Override 127 | public SubArray getRaw() { 128 | return raw; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/logger/AbstractLogger.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.logger; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import com.northeastern.edu.simpledb.backend.utils.Parser; 5 | import com.northeastern.edu.simpledb.common.Error; 6 | 7 | import java.io.File; 8 | import java.io.FileNotFoundException; 9 | import java.io.IOException; 10 | import java.io.RandomAccessFile; 11 | import java.nio.ByteBuffer; 12 | import java.nio.channels.FileChannel; 13 | 14 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX; 15 | 16 | abstract class AbstractLogger { 17 | abstract void log(byte[] data); 18 | abstract void truncate(long x) throws IOException; 19 | abstract byte[] next(); 20 | abstract void rewind(); 21 | abstract void close(); 22 | 23 | // create log file based on path, initialize logger 24 | public static Logger create(String path) { 25 | File f = new File(path + LOG_SUFFIX); 26 | try { 27 | if (!f.createNewFile()) { 28 | Panic.panic(Error.FileExistsException); 29 | } 30 | } catch (IOException e) { 31 | throw new RuntimeException(e); 32 | } 33 | 34 | FileChannel fc = null; 35 | RandomAccessFile raf = null; 36 | try { 37 | raf = new RandomAccessFile(f, "rw"); 38 | fc = raf.getChannel(); 39 | } catch (FileNotFoundException e) { 40 | Panic.panic(e); 41 | } 42 | 43 | ByteBuffer buf = ByteBuffer.wrap(Parser.int2Byte(0)); 44 | 45 | try { 46 | fc.position(0); 47 | fc.write(buf); 48 | } catch (IOException e) { 49 | Panic.panic(e); 50 | } 51 | 52 | return new Logger(raf, fc, 0); 53 | } 54 | 55 | // open log file based on path, init, check and format logger 56 | public static Logger open(String path) { 57 | File f = new File(path + LOG_SUFFIX); 58 | 59 | if (!f.exists()) Panic.panic(Error.FileNotExistsException); 60 | if (!f.canRead() || !f.canWrite()) Panic.panic(Error.FileCannotRWException); 61 | 62 | FileChannel fc = null; 63 | RandomAccessFile raf = null; 64 | try { 65 | raf = new RandomAccessFile(f, "rw"); 66 | fc = raf.getChannel(); 67 | } catch (FileNotFoundException e) { 68 | Panic.panic(e); 69 | } 70 | 71 | Logger logger = new Logger(raf, fc); 72 | logger.init(); 73 | 74 | return logger; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/logger/Logger.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.logger; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.backend.utils.Panic; 5 | import com.northeastern.edu.simpledb.backend.utils.Parser; 6 | import com.northeastern.edu.simpledb.common.Error; 7 | 8 | import java.io.IOException; 9 | import java.io.RandomAccessFile; 10 | import java.nio.ByteBuffer; 11 | import java.nio.channels.FileChannel; 12 | import java.util.Arrays; 13 | import java.util.concurrent.locks.Lock; 14 | import java.util.concurrent.locks.ReentrantLock; 15 | 16 | import static com.northeastern.edu.simpledb.common.Error.BadLogFileException; 17 | 18 | public class Logger extends AbstractLogger{ 19 | 20 | // a prime number for calculating check sum 21 | private static final int SEED = 13331; 22 | 23 | // starting position of attribute `size` in a log 24 | private static final int OF_SIZE = 0; 25 | 26 | // starting position of attribute `check sum` in a log 27 | private static final int OF_CHECKSUM = OF_SIZE + 4; 28 | 29 | // starting position of attribute `data` in a log 30 | private static final int OF_DATA = OF_CHECKSUM + 4; 31 | 32 | // standard suffix of log file 33 | public static final String LOG_SUFFIX = ".log"; 34 | 35 | private Lock lock; 36 | private RandomAccessFile raf; 37 | private FileChannel fc; 38 | 39 | // the position of the pointer of logger 40 | private long position; 41 | 42 | // the size of log file, it will be updated when `open()` from disk 43 | private long fileSize; 44 | 45 | // the check sum of log file 46 | private int xCheckSum; 47 | 48 | public Logger(RandomAccessFile raf, FileChannel fc) { 49 | this.raf = raf; 50 | this.fc = fc; 51 | this.lock = new ReentrantLock(); 52 | } 53 | 54 | public Logger(RandomAccessFile raf, FileChannel fc, int xCheckSum) { 55 | this.raf = raf; 56 | this.fc = fc; 57 | this.xCheckSum = xCheckSum; 58 | this.lock = new ReentrantLock(); 59 | } 60 | 61 | /** 62 | * load check sum of log file, update fileSize as same as the 63 | * length of log file, and validate log file using check sum 64 | */ 65 | void init() { 66 | long size = 0; 67 | try { 68 | size = raf.length(); 69 | } catch (IOException e) { 70 | Panic.panic(e); 71 | } 72 | 73 | if (size < 4) Panic.panic(BadLogFileException); 74 | 75 | // read size from log file 76 | ByteBuffer buf = ByteBuffer.allocate(4); 77 | try { 78 | fc.position(0); 79 | fc.read(buf); 80 | } catch (IOException e) { 81 | Panic.panic(e); 82 | } 83 | 84 | int xChecksum = Parser.parseInt(buf.array()); 85 | this.fileSize = size; 86 | this.xCheckSum = xChecksum; 87 | 88 | checkAndRemoveTail(); 89 | } 90 | 91 | // validate log file, remove invalid tail 92 | private void checkAndRemoveTail() { 93 | rewind(); 94 | 95 | int xCheck = 0; 96 | while (true) { 97 | // get the next log 98 | byte[] log = internNext(); 99 | // calculate xChecksum accumulate 100 | if (log == null) break; 101 | xCheck = calChecksum(xCheck, log); 102 | } 103 | 104 | if (xCheck != xCheckSum) Panic.panic(Error.BadLogFileException); 105 | 106 | try { 107 | truncate(position); 108 | } catch (Exception e) { 109 | Panic.panic(e); 110 | } 111 | 112 | try { 113 | raf.seek(position); 114 | } catch (Exception e) { 115 | Panic.panic(e); 116 | } 117 | rewind(); 118 | } 119 | 120 | // calculate check sum of log file, then flush to disk 121 | private void updateChecksum(byte[] log) { 122 | this.xCheckSum = calChecksum(this.xCheckSum, log); 123 | ByteBuffer buf = ByteBuffer.wrap(Parser.int2Byte(this.xCheckSum)); 124 | try { 125 | fc.position(0); 126 | fc.write(buf); 127 | fc.force(false); 128 | } catch (IOException e) { 129 | Panic.panic(e); 130 | } 131 | } 132 | 133 | // wrap data as a log 134 | private byte[] wrapLog(byte[] data) { 135 | byte[] checksum = Parser.int2Byte(calChecksum(0, data)); 136 | byte[] size = Parser.int2Byte(data.length); 137 | return Bytes.concat(size, checksum, data); 138 | } 139 | 140 | /** 141 | * handle bad tail in two cases 142 | * case1: position + OF_DATA > raf.length || position + OF_DATA + data size > raf.length 143 | * case2: check sum1 != check sum2 144 | */ 145 | private byte[] internNext() { 146 | if (position + OF_DATA >= fileSize) return null; 147 | 148 | // step1 check size of a log: [size][checksum][data] 149 | ByteBuffer tmp = ByteBuffer.allocate(4); 150 | 151 | try { 152 | fc.position(position); 153 | fc.read(tmp); 154 | } catch (IOException e) { 155 | Panic.panic(e); 156 | } 157 | 158 | int size = Parser.parseInt(tmp.array()); 159 | if (position + OF_DATA + size > fileSize) return null; 160 | 161 | // step2 read whole check sum and data, then validate check sum 162 | ByteBuffer buf = ByteBuffer.allocate(OF_DATA + size); 163 | 164 | try { 165 | fc.position(position); 166 | fc.read(buf); 167 | } catch (Exception e) { 168 | Panic.panic(e); 169 | } 170 | 171 | byte[] log = buf.array(); 172 | int checksum1 = calChecksum(0, Arrays.copyOfRange(log, OF_DATA, log.length)); 173 | int checksum2 = Parser.parseInt(Arrays.copyOfRange(log, OF_CHECKSUM, OF_DATA)); 174 | if (checksum1 != checksum2) return null; 175 | 176 | // step3 update position 177 | position += log.length; 178 | return log; 179 | } 180 | 181 | 182 | // calculate check sum based on xCheck and a log 183 | private int calChecksum(int xCheck, byte[] log) { 184 | for (byte b : log) { 185 | xCheck = xCheck * SEED + b; 186 | } 187 | return xCheck; 188 | } 189 | 190 | @Override 191 | public void truncate(long position) throws IOException { 192 | lock.lock(); 193 | try { 194 | fc.truncate(position); 195 | } finally { 196 | lock.unlock(); 197 | } 198 | } 199 | 200 | // wrap data to log, write a log to log file, then update check sum of log file 201 | @Override 202 | public void log(byte[] data) { 203 | // step1 build log and its format is like `[size][checksum][data]` 204 | byte[] log = wrapLog(data); 205 | 206 | // step2 write log to file 207 | ByteBuffer buf = ByteBuffer.wrap(log); 208 | lock.lock(); 209 | try { 210 | fc.position(fc.size()); 211 | fc.write(buf); 212 | } catch (IOException e) { 213 | Panic.panic(e); 214 | } finally { 215 | lock.unlock(); 216 | } 217 | 218 | // step3 update check sum 219 | updateChecksum(log); 220 | } 221 | 222 | @Override 223 | public void rewind() { 224 | position = 4; 225 | } 226 | 227 | @Override 228 | public byte[] next() { 229 | lock.lock(); 230 | try { 231 | byte[] log = internNext(); 232 | if (log == null) return null; 233 | return Arrays.copyOfRange(log, OF_DATA, log.length); 234 | } finally { 235 | lock.unlock(); 236 | } 237 | } 238 | 239 | @Override 240 | public void close() { 241 | try { 242 | fc.close(); 243 | raf.close(); 244 | } catch (IOException e) { 245 | Panic.panic(e); 246 | } 247 | } 248 | 249 | public long getPosition() { 250 | return this.position; 251 | } 252 | 253 | public FileChannel getFc() { 254 | return this.fc; 255 | }; 256 | 257 | } 258 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/page/AbstractPage.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.page; 2 | 3 | public abstract class AbstractPage { 4 | 5 | abstract void lock(); 6 | 7 | abstract void unlock(); 8 | 9 | abstract void release(); 10 | 11 | abstract void setDirty(boolean dirty); 12 | 13 | abstract boolean isDirty(); 14 | 15 | abstract int getPageNumber(); 16 | 17 | abstract byte[] getData(); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/page/L1Page.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.page; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | import com.northeastern.edu.simpledb.backend.utils.RandomUtil; 5 | 6 | import java.util.Arrays; 7 | 8 | /** 9 | * L1 Page 10 | * when db start up, write random bytes from OF_VC to (OF_VC + LEN_VC - 1) byte 11 | * when db close, copy random bytes from (OF_VC + LEN_VC) to (OF_VC + 2 * LEN_VC - 1) byte 12 | * used to determine whether the database was shut down normally in the last time 13 | */ 14 | public class L1Page { 15 | 16 | private final static int OF_VC = 100; 17 | 18 | private final static int LEN_VC = 8; 19 | 20 | public static byte[] initRaw() { 21 | byte[] raw = new byte[PageCache.PAGE_SIZE]; 22 | setVcOpen(raw); 23 | return raw; 24 | } 25 | 26 | public static void setVcOpen(Page page) { 27 | page.setDirty(true); 28 | setVcOpen(page.getData()); 29 | } 30 | 31 | private static void setVcOpen(byte[] raw) { 32 | System.arraycopy(RandomUtil.randomBytes(LEN_VC), 0, raw, OF_VC, LEN_VC); 33 | } 34 | 35 | public static void setVcClose(Page page) { 36 | page.setDirty(true); 37 | setVcClose(page.getData()); 38 | } 39 | 40 | private static void setVcClose(byte[] raw) { 41 | System.arraycopy(raw, OF_VC, raw, OF_VC + LEN_VC, LEN_VC); 42 | } 43 | 44 | public static boolean checkVc(Page pg) { 45 | return checkVc(pg.getData()); 46 | } 47 | 48 | private static boolean checkVc(byte[] raw) { 49 | return Arrays.equals(Arrays.copyOfRange(raw, OF_VC, OF_VC + LEN_VC), Arrays.copyOfRange(raw, OF_VC + LEN_VC, OF_VC + 2 * LEN_VC)); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/page/Page.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.page; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | 5 | import java.util.concurrent.locks.Lock; 6 | import java.util.concurrent.locks.ReentrantLock; 7 | 8 | public class Page extends AbstractPage { 9 | 10 | private int pageNumber; 11 | private byte[] data; 12 | private boolean dirty; 13 | private Lock lock; 14 | private PageCache pageCache; 15 | 16 | 17 | public Page(int pageNumber, byte[] data, PageCache pc) { 18 | this.pageNumber = pageNumber; 19 | this.data = data; 20 | this.pageCache = pc; 21 | this.lock = new ReentrantLock(); 22 | } 23 | 24 | @Override 25 | public void lock() { 26 | lock.lock(); 27 | } 28 | 29 | @Override 30 | public void unlock() { 31 | lock.unlock(); 32 | } 33 | 34 | @Override 35 | public void release() { 36 | pageCache.release(this); 37 | } 38 | 39 | @Override 40 | public void setDirty(boolean dirty) { 41 | this.dirty = dirty; 42 | } 43 | 44 | @Override 45 | public boolean isDirty() { 46 | return this.dirty; 47 | } 48 | 49 | @Override 50 | public int getPageNumber() { 51 | return pageNumber; 52 | } 53 | 54 | @Override 55 | public byte[] getData() { 56 | return data; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/page/SecondaryPage.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.page; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | import com.northeastern.edu.simpledb.backend.utils.Parser; 5 | 6 | import java.util.Arrays; 7 | 8 | /** 9 | * Secondary Page 10 | * starting with a 2 bytes unsigned number representing the 11 | * offset of the free location in this pge. the rest is the 12 | * actual stored data 13 | */ 14 | public class SecondaryPage { 15 | 16 | private static final short OF_FREE = 0; 17 | private static final short OF_DATA = 2; 18 | public static final int MAX_FREE_SPACE = PageCache.PAGE_SIZE - OF_DATA; 19 | 20 | // init a raw data as secondary page [Offset: 2bytes][Data] 21 | public static byte[] initRaw() { 22 | byte[] raw = new byte[PageCache.PAGE_SIZE]; 23 | setFSO(raw, OF_DATA); 24 | return raw; 25 | } 26 | 27 | // insert raw into page, return insertion position 28 | public static short insert(Page page, byte[] raw) { 29 | page.setDirty(true); 30 | // get free space offset 31 | short offset = getFSO(page.getData()); 32 | System.arraycopy(raw, 0, page.getData(), offset, raw.length); 33 | // update free space offset 34 | setFSO(page.getData(), (short)(offset + raw.length)); 35 | return offset; 36 | } 37 | 38 | private static void setFSO(byte[] raw, short ofData) { 39 | System.arraycopy(Parser.short2Byte(ofData), 0, raw, OF_FREE, OF_DATA); 40 | } 41 | 42 | private static short getFSO(byte[] raw) { 43 | return Parser.parseShort(Arrays.copyOfRange(raw, OF_FREE, OF_DATA)); 44 | } 45 | 46 | public static short getFSO(Page page) { 47 | return getFSO(page.getData()); 48 | } 49 | 50 | // comes in handy when calculating free space for page index 51 | public static int getFreeSpace(Page page) { 52 | return PageCache.PAGE_SIZE - (int) getFSO(page.getData()); 53 | } 54 | 55 | // recover insert statement from log file, update FSO of page if needed 56 | public static void recoverInsert(Page page, byte[] raw, short offset) { 57 | page.setDirty(true); 58 | System.arraycopy(raw, 0, page.getData(), offset, raw.length); 59 | 60 | short rawFSO = getFSO(page.getData()); 61 | if (rawFSO < offset + raw.length) setFSO(page.getData(), (short) (offset + raw.length)); 62 | } 63 | 64 | // recover update statement from log file 65 | public static void recoverUpdate(Page page, byte[] raw, short offset) { 66 | page.setDirty(true); 67 | System.arraycopy(raw, 0, page.getData(), offset, raw.length); 68 | } 69 | 70 | } 71 | 72 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/pageIndex/PageIndex.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.pageIndex; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.concurrent.locks.Lock; 8 | import java.util.concurrent.locks.ReentrantLock; 9 | 10 | /** 11 | * Page Index 12 | * the page index caches the free space of each page. It is 13 | * used to quickly find a page with suitable space when the 14 | * upper module performs an insertion operation without checking 15 | * the information of each page from the disk or cache 16 | */ 17 | public class PageIndex { 18 | 19 | private static final int INTERVALS_NO = 40; 20 | private static final int THRESHOLD = PageCache.PAGE_SIZE / INTERVALS_NO; 21 | 22 | private Lock lock; 23 | 24 | /** 25 | * entry in Map 26 | * key: the number of free slot : Integer 27 | * value: all of PageInfo have the number of free slot : List 28 | */ 29 | private List[] lists; 30 | 31 | @SuppressWarnings("unchecked") 32 | public PageIndex() { 33 | this.lock = new ReentrantLock(); 34 | this.lists = new List[INTERVALS_NO + 1]; 35 | for (int i = 0; i < INTERVALS_NO + 1; i++) { 36 | lists[i] = new ArrayList<>(); 37 | } 38 | } 39 | 40 | 41 | public void add(int pageNumber, int freeSpace) { 42 | int number = freeSpace / THRESHOLD; // how many free slot this page has? 43 | lists[number].add(new PageInfo(pageNumber, freeSpace)); 44 | } 45 | 46 | /** 47 | * calculating how many free slot needed, then 48 | * accessing the next one, having more slot and 49 | * ensuring the data won't across two pages. 50 | */ 51 | public PageInfo select(int spaceSize) { 52 | int number = spaceSize / THRESHOLD; // how many free slot need? 53 | if (number < INTERVALS_NO) number++; // rounded up 54 | 55 | // iterate over lists finding the first page doesn't acquire by other thread 56 | while (number <= INTERVALS_NO) { 57 | if (lists[number].size() == 0) { 58 | number++; 59 | continue; 60 | } 61 | if (lists[number].size() > 0) return lists[number].remove(0); 62 | } 63 | return null; 64 | } 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/dm/pageIndex/PageInfo.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.pageIndex; 2 | 3 | public class PageInfo { 4 | public int pageNumber; 5 | public int freeSpace; 6 | 7 | public PageInfo(int pageNumber, int freeSpace) { 8 | this.pageNumber = pageNumber; 9 | this.freeSpace = freeSpace; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/im/BPlusTree.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.im; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.SubArray; 4 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.utils.Parser; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.concurrent.locks.Lock; 12 | import java.util.concurrent.locks.ReentrantLock; 13 | 14 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID; 15 | 16 | public class BPlusTree { 17 | 18 | private static final int UID_SIZE = 8; 19 | DataManger dm; 20 | long bootUid; 21 | DataItem bootDataItem; 22 | Lock bootLock; 23 | 24 | 25 | /** 26 | * storing uid of insertion of nil root raw, aka rootUid 27 | * return the uid of insertion of uid of insertion of nil root raw, aka bootUid 28 | */ 29 | public static long create(DataManger dm) throws Exception { 30 | byte[] rawRoot = Node.newNilRootRaw(); // get nil root raw: [1,0,0,0,0,0,0,0...] 31 | long rootUid = dm.insert(SUPER_XID, rawRoot); // store nil root raw, and return uid of insertion of nil root raw 32 | return dm.insert(SUPER_XID, Parser.long2Byte(rootUid)); // store uid, return uid of insertion of uid of nil root raw 33 | } 34 | 35 | // loading rootUid by bootUid 36 | public static BPlusTree load(long bootUid, DataManger dm) throws Exception { // 37 | DataItem bootDataItem = dm.read(bootUid); 38 | assert bootDataItem != null; 39 | BPlusTree bPlusTree = new BPlusTree(); 40 | bPlusTree.bootDataItem = bootDataItem; 41 | bPlusTree.dm = dm; 42 | bPlusTree.bootUid = bootUid; 43 | bPlusTree.bootLock = new ReentrantLock(); 44 | return bPlusTree; 45 | } 46 | 47 | // retrieving rootUid from the data of bootDataItem 48 | private long rootUid() { 49 | bootLock.lock(); 50 | try { 51 | SubArray subArray = bootDataItem.data(); 52 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start, subArray.start + UID_SIZE)); 53 | } finally { 54 | bootLock.unlock(); 55 | } 56 | } 57 | 58 | /** 59 | * [newRootUid] 60 | * / \ 61 | * [rootUid] [uid] 62 | */ 63 | private void updateRootUid(long left, long right, long rightKey) throws Exception { 64 | byte[] rootRaw = Node.newRootRaw(left, right, rightKey); 65 | long newRootUid = dm.insert(SUPER_XID, rootRaw); 66 | bootDataItem.before(); 67 | SubArray diRaw = bootDataItem.data(); 68 | System.arraycopy(Parser.long2Byte(newRootUid), 0, diRaw.raw, diRaw.start, UID_SIZE); 69 | bootDataItem.after(SUPER_XID); 70 | } 71 | 72 | public void insert(long key, long uid) throws Exception { 73 | long rootUid = rootUid(); 74 | InsertRes res = insert(rootUid, uid, key); 75 | assert res != null; 76 | if (res.newNode != 0) updateRootUid(rootUid, res.newNode, res.newKey); 77 | } 78 | 79 | class InsertRes { 80 | long newNode; 81 | long newKey; 82 | } 83 | 84 | /** 85 | * insert uid and key into B+ Tree based on nodeUid 86 | */ 87 | private InsertRes insert(long nodeUid, long uid, long key) throws Exception { 88 | Node node = Node.loadNode(this, nodeUid); 89 | boolean isLeaf = node.isLeaf(); 90 | node.release(); 91 | 92 | // in two situations, insertAndSplit will be invoked 93 | InsertRes res = null; 94 | if (isLeaf) { 95 | // 1. insert to a leaf node 96 | res = insertAndSplit(nodeUid, uid, key); 97 | } else { 98 | long next = searchNext(nodeUid, key); // find the first son has larger key than input key 99 | InsertRes insertRes = insert(next, uid, key); 100 | if (insertRes.newNode != 0) { 101 | // 2. after splitting, a new node created, and this need to be inserted into b+tree based on the current node 102 | res = insertAndSplit(nodeUid, insertRes.newNode, insertRes.newKey); 103 | } else { 104 | res = new InsertRes(); 105 | } 106 | } 107 | return res; 108 | } 109 | 110 | private InsertRes insertAndSplit(long nodeUid, long uid, long key) throws Exception { 111 | for (;;) { 112 | Node node = Node.loadNode(this, nodeUid); 113 | Node.InsertAndSplitRes insertAndSplitRes = node.insertAndSplit(uid, key); 114 | if (insertAndSplitRes.siblingUid != 0) { 115 | nodeUid = insertAndSplitRes.siblingUid; 116 | } else { 117 | InsertRes insertRes = new InsertRes(); 118 | insertRes.newKey = insertAndSplitRes.newKey; 119 | insertRes.newNode = insertAndSplitRes.newSon; 120 | return insertRes; 121 | } 122 | } 123 | } 124 | 125 | // search in node starting from root util reaching leaf 126 | private long searchLeaf(long nodeUid, long key) throws Exception { 127 | Node node = Node.loadNode(this, nodeUid); 128 | // System.out.println("node = " + node); for test purpose 129 | boolean isLeaf = node.isLeaf(); 130 | node.release(); 131 | 132 | if (isLeaf) { 133 | return nodeUid; 134 | } else { 135 | long next = searchNext(nodeUid, key); 136 | return searchLeaf(next, key); 137 | } 138 | } 139 | 140 | 141 | // search uid of key in the current node, if not found return uid of the sibling 142 | private long searchNext(long nodeUid, long key) throws Exception{ 143 | for(;;) { 144 | Node node = Node.loadNode(this, nodeUid); 145 | Node.SearchNextRes res = node.searchNext(key); 146 | node.release(); 147 | if (res.uid != 0) return res.uid; 148 | nodeUid = res.siblingUid; 149 | } 150 | } 151 | 152 | // search() -> searchRange() -> searchLeaf() -> searchNext() 153 | public List search(long key) throws Exception { 154 | return searchRange(key, key); 155 | } 156 | 157 | public List searchRange(long leftKey, long rightKey) throws Exception { 158 | long rootUid = rootUid(); 159 | long leftUid = searchLeaf(rootUid, leftKey); 160 | List uids = new ArrayList<>(); 161 | for (;;) { // searching all the siblings starting from the given leaf node and collecting all uids 162 | Node leaf = Node.loadNode(this, leftUid); 163 | Node.LeafSearchRangeRes res = leaf.leafSearchRangeRes(leftKey, rightKey); 164 | leaf.release(); 165 | uids.addAll(res.uids); 166 | if (res.siblingUid == 0) break; 167 | else leftUid = res.siblingUid; 168 | } 169 | return uids; 170 | } 171 | 172 | public void close() { 173 | bootDataItem.release(); 174 | } 175 | 176 | } 177 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/im/Node.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.im; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.SubArray; 4 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 5 | import com.northeastern.edu.simpledb.backend.utils.Parser; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID; 12 | 13 | public class Node { 14 | 15 | // Node will be stored as a DataItem 16 | // [Valid][Size][LeafFlag][NumberOfKeys][SiblingUid][Son0][Key0][Son1][Key1]...[SonN][KeyN] 17 | 18 | private static final int SON_SIZE = 8; 19 | 20 | private static final int KEY_SIZE = 8; 21 | 22 | private static final int NUMBER_OF_KEYS_SIZE = 2; 23 | 24 | private static final int SIBLING_UID_SIZE = 8; 25 | 26 | private static final int LEAF_FLAG_SIZE = 1; 27 | 28 | private static final int IS_LEAF_OFFSET = 0; 29 | 30 | private static final int NUMBER_OF_KEYS_OFFSET = IS_LEAF_OFFSET + LEAF_FLAG_SIZE; 31 | private static final int SIBLING_OFFSET = NUMBER_OF_KEYS_OFFSET + NUMBER_OF_KEYS_SIZE; 32 | private static final int NODE_HEADER_SIZE = SIBLING_OFFSET + SIBLING_UID_SIZE; 33 | private static final int BALANCE_NUMBER = 3; 34 | 35 | private static final int NODE_SIZE = NODE_HEADER_SIZE + (SON_SIZE + KEY_SIZE) * (BALANCE_NUMBER * 2 + 2); 36 | 37 | BPlusTree tree; 38 | DataItem dataItem; 39 | SubArray raw; 40 | long uid; 41 | 42 | static void setRawIsLeaf(SubArray raw, boolean isLeaf) { 43 | raw.raw[raw.start + IS_LEAF_OFFSET] = isLeaf ? (byte) 1 : (byte) 0; 44 | } 45 | 46 | static boolean getRawIfLeaf(SubArray raw) { 47 | return raw.raw[raw.start + IS_LEAF_OFFSET] == (byte) 1; 48 | } 49 | 50 | static void setRawNoKeys(SubArray raw, int numberOfKeys) { 51 | System.arraycopy(Parser.short2Byte((short) numberOfKeys), 0, raw.raw, raw.start + NUMBER_OF_KEYS_OFFSET, NUMBER_OF_KEYS_SIZE); 52 | } 53 | 54 | static int getRawNoKeys(SubArray raw) { 55 | return (int) Parser.parseShort(Arrays.copyOfRange(raw.raw, raw.start + NUMBER_OF_KEYS_OFFSET, raw.start + NUMBER_OF_KEYS_OFFSET + NUMBER_OF_KEYS_SIZE)); 56 | } 57 | 58 | static void setRawSibling(SubArray raw, long numberOfSiblings) { 59 | System.arraycopy(Parser.long2Byte(numberOfSiblings), 0, raw.raw, raw.start + SIBLING_OFFSET, SIBLING_UID_SIZE); 60 | } 61 | 62 | static long getRawSibling(SubArray raw) { 63 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, raw.start + SIBLING_OFFSET, raw.start + SIBLING_OFFSET + SIBLING_UID_SIZE)); 64 | } 65 | 66 | static void setRawKthSon(SubArray raw, long uid, int kth) { 67 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE); 68 | System.arraycopy(Parser.long2Byte(uid), 0, raw.raw, offset, SON_SIZE); 69 | } 70 | 71 | static long getRawKthSon(SubArray raw, int kth) { 72 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE); 73 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, offset, offset + SON_SIZE)); 74 | } 75 | 76 | static void setRawKthKey(SubArray raw, long key, int kth) { 77 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE) + SON_SIZE; 78 | System.arraycopy(Parser.long2Byte(key), 0, raw.raw, offset, KEY_SIZE); 79 | } 80 | 81 | static long getRawKthKey(SubArray raw, int kth) { 82 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE) + SON_SIZE; 83 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, offset, offset + KEY_SIZE)); 84 | } 85 | 86 | static void shiftRawKth(SubArray raw, int kth) { 87 | int begin = raw.start + NODE_HEADER_SIZE + (kth + 1) * (SON_SIZE + KEY_SIZE); 88 | int end = raw.start + NODE_SIZE - 1; 89 | for (int i = end; i >= begin; i--) { 90 | raw.raw[i] = raw.raw[i - (SON_SIZE + KEY_SIZE)]; 91 | } 92 | } 93 | 94 | static void copyRawFromKth(SubArray from, SubArray to, int kth) { 95 | int offset = from.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE); 96 | System.arraycopy(from.raw, offset, to.raw, to.start + NODE_HEADER_SIZE, from.end - offset); 97 | } 98 | 99 | static byte[] newRootRaw(long left, long right, long key) { 100 | SubArray raw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE); 101 | 102 | setRawIsLeaf(raw, false); 103 | setRawNoKeys(raw, 2); 104 | setRawSibling(raw, 0); 105 | setRawKthSon(raw, left, 0); 106 | setRawKthKey(raw, key, 0); 107 | setRawKthSon(raw, right, 1); 108 | setRawKthKey(raw, Long.MAX_VALUE, 1); 109 | 110 | return raw.raw; 111 | } 112 | 113 | static byte[] newNilRootRaw() { 114 | SubArray raw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE); 115 | setRawIsLeaf(raw, true); 116 | setRawNoKeys(raw, 0); 117 | setRawSibling(raw, 0); 118 | 119 | return raw.raw; 120 | } 121 | 122 | static Node loadNode(BPlusTree bTree, long uid) throws Exception { 123 | DataItem dataItem = bTree.dm.read(uid); 124 | assert dataItem != null; 125 | Node node = new Node(); 126 | node.tree = bTree; 127 | node.dataItem = dataItem; 128 | node.raw = dataItem.data(); 129 | node.uid = uid; 130 | return node; 131 | } 132 | 133 | public void release() { 134 | dataItem.release(); 135 | } 136 | 137 | public boolean isLeaf() { 138 | dataItem.rLock(); 139 | try { 140 | return getRawIfLeaf(raw); 141 | } finally { 142 | dataItem.rUnLock(); 143 | } 144 | } 145 | 146 | 147 | class SearchNextRes { 148 | long uid; 149 | long siblingUid; 150 | } 151 | 152 | public SearchNextRes searchNext(long key) { 153 | dataItem.rLock(); 154 | try { 155 | SearchNextRes res = new SearchNextRes(); 156 | int noKeys = getRawNoKeys(raw); 157 | for (int i = 0; i < noKeys; i++) { 158 | long ik = getRawKthKey(raw, i); 159 | if (key < ik) { 160 | res.uid = getRawKthSon(raw, i); 161 | res.siblingUid = 0; 162 | return res; 163 | } 164 | } 165 | res.uid = 0; 166 | res.siblingUid = getRawSibling(raw); 167 | return res; 168 | } finally { 169 | dataItem.rUnLock(); 170 | } 171 | } 172 | 173 | class LeafSearchRangeRes { 174 | List uids; 175 | long siblingUid; 176 | } 177 | 178 | public LeafSearchRangeRes leafSearchRangeRes(long leftKey, long rightKey) { 179 | dataItem.rLock(); 180 | try { 181 | int noKeys = getRawNoKeys(raw); 182 | int kth = 0; 183 | while (kth < noKeys) { 184 | long ik = getRawKthKey(raw, kth); 185 | if (ik >= leftKey) { 186 | break; 187 | } 188 | kth++; 189 | } 190 | List uids = new ArrayList<>(); 191 | while (kth < noKeys) { 192 | long ik = getRawKthKey(raw, kth); 193 | if (ik <= rightKey) { 194 | uids.add(getRawKthSon(raw, kth)); 195 | kth ++; 196 | } else break; 197 | } 198 | long siblingUid = 0; 199 | if (kth == noKeys) { 200 | siblingUid = getRawSibling(raw); 201 | } 202 | LeafSearchRangeRes res = new LeafSearchRangeRes(); 203 | res.uids = uids; 204 | res.siblingUid = siblingUid; 205 | return res; 206 | } finally { 207 | dataItem.rUnLock(); 208 | } 209 | } 210 | 211 | class InsertAndSplitRes { 212 | long siblingUid; 213 | long newSon; 214 | long newKey; 215 | } 216 | 217 | public InsertAndSplitRes insertAndSplit(long uid, long key) throws Exception { 218 | boolean success = false; 219 | Exception err = null; 220 | InsertAndSplitRes insertAndSplitRes = new InsertAndSplitRes(); 221 | 222 | dataItem.before(); 223 | try { 224 | success = insert(uid, key); 225 | if (!success) { 226 | insertAndSplitRes.siblingUid = getRawSibling(raw); 227 | return insertAndSplitRes; 228 | } 229 | if (needSplit()) { 230 | try { 231 | SplitRes split = split(); 232 | insertAndSplitRes.newSon = split.newSon; 233 | insertAndSplitRes.newKey = split.newKey; 234 | return insertAndSplitRes; 235 | } catch (Exception e) { 236 | err = e; 237 | throw e; 238 | } 239 | } else return insertAndSplitRes; 240 | } finally { 241 | if (err == null && success) dataItem.after(SUPER_XID); 242 | else dataItem.unBefore(); 243 | } 244 | } 245 | 246 | private boolean insert(long uid, long key) { 247 | int noKeys = getRawNoKeys(raw); 248 | int kth = 0; 249 | while(kth < noKeys) { 250 | long ik = getRawKthKey(raw, kth); 251 | if(ik < key) { 252 | kth ++; 253 | } else { 254 | break; 255 | } 256 | } 257 | if(kth == noKeys && getRawSibling(raw) != 0) return false; 258 | 259 | if(getRawIfLeaf(raw)) { 260 | shiftRawKth(raw, kth); 261 | setRawKthKey(raw, key, kth); 262 | setRawKthSon(raw, uid, kth); 263 | setRawNoKeys(raw, noKeys + 1); 264 | } else { 265 | long kk = getRawKthKey(raw, kth); 266 | setRawKthKey(raw, key, kth); 267 | shiftRawKth(raw, kth + 1); 268 | setRawKthKey(raw, kk, kth + 1); 269 | setRawKthSon(raw, uid, kth + 1); 270 | setRawNoKeys(raw, noKeys + 1); 271 | } 272 | return true; 273 | } 274 | 275 | private boolean needSplit() { 276 | return BALANCE_NUMBER * 2 == getRawNoKeys(raw); 277 | } 278 | 279 | class SplitRes { 280 | long newSon; 281 | long newKey; 282 | } 283 | 284 | private SplitRes split() throws Exception{ 285 | System.out.println("split..."); 286 | SubArray nodeRaw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE); 287 | setRawIsLeaf(nodeRaw, getRawIfLeaf(raw)); 288 | setRawNoKeys(nodeRaw, BALANCE_NUMBER); 289 | setRawSibling(nodeRaw, getRawSibling(raw)); 290 | copyRawFromKth(raw, nodeRaw, BALANCE_NUMBER); 291 | long son = tree.dm.insert(SUPER_XID, nodeRaw.raw); 292 | setRawNoKeys(raw, BALANCE_NUMBER); 293 | setRawSibling(raw, son); 294 | 295 | SplitRes splitRes = new SplitRes(); 296 | splitRes.newSon = son; 297 | splitRes.newKey = getRawKthKey(nodeRaw, 0); 298 | return splitRes; 299 | } 300 | 301 | @Override 302 | public String toString() { 303 | StringBuilder sb = new StringBuilder(); 304 | sb.append("Is leaf: ").append(getRawIfLeaf(raw)).append("\n"); 305 | int KeyNumber = getRawNoKeys(raw); 306 | sb.append("KeyNumber: ").append(KeyNumber).append("\n"); 307 | sb.append("sibling: ").append(getRawSibling(raw)).append("\n"); 308 | for(int i = 0; i < KeyNumber; i ++) { 309 | sb.append("son: ").append(getRawKthSon(raw, i)).append(", key: ").append(getRawKthKey(raw, i)).append("\n"); 310 | } 311 | return sb.toString(); 312 | } 313 | } 314 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/Tokenizer.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser; 2 | 3 | import com.northeastern.edu.simpledb.common.Error; 4 | 5 | //public class Tokenizer { 6 | // private enum State { 7 | // INIT, SYMBOL, QUOTE, TOKEN 8 | // } 9 | // 10 | // private byte[] stat; 11 | // private int pos; 12 | // private String currentToken; 13 | // private boolean flushToken; 14 | // private Exception err; 15 | // private State currentState; 16 | // 17 | // public Tokenizer(byte[] stat) { 18 | // this.stat = stat; 19 | // this.pos = 0; 20 | // this.currentToken = ""; 21 | // this.flushToken = true; 22 | // this.currentState = State.INIT; 23 | // } 24 | // 25 | // public String peek() throws Exception { 26 | // if (err != null) { 27 | // throw err; 28 | // } 29 | // if (flushToken) { 30 | // String token = null; 31 | // try { 32 | // token = next(); 33 | // } catch (Exception e) { 34 | // err = e; 35 | // throw e; 36 | // } 37 | // currentToken = token; 38 | // flushToken = false; 39 | // } 40 | // return currentToken; 41 | // } 42 | // 43 | // public void pop() { 44 | // flushToken = true; 45 | // } 46 | // 47 | // public byte[] errStat() { 48 | // byte[] res = new byte[stat.length + 3]; 49 | // System.arraycopy(stat, 0, res, 0, pos); 50 | // System.arraycopy("<< ".getBytes(), 0, res, pos, 3); 51 | // System.arraycopy(stat, pos, res, pos + 3, stat.length - pos); 52 | // return res; 53 | // } 54 | // 55 | // private void popByte() { 56 | // pos++; 57 | // if (pos > stat.length) { 58 | // pos = stat.length; 59 | // } 60 | // } 61 | // 62 | // private Byte peekByte() { 63 | // if (pos == stat.length) { 64 | // return null; 65 | // } 66 | // return stat[pos]; 67 | // } 68 | // 69 | // private String next() throws Exception { 70 | // if (err != null) { 71 | // throw err; 72 | // } 73 | // 74 | // while (true) { 75 | // switch (currentState) { 76 | // case INIT: 77 | // currentState = nextState(); 78 | // break; 79 | // case SYMBOL: 80 | // return nextSymbolState(); 81 | // case QUOTE: 82 | // return nextQuoteState(); 83 | // case TOKEN: 84 | // return nextTokenState(); 85 | // } 86 | // } 87 | // } 88 | // 89 | // private State nextState() throws Exception { 90 | // Byte b = peekByte(); 91 | // if (b == null) { 92 | // return State.INIT; 93 | // } else if (isSymbol(b)) { 94 | // return State.SYMBOL; 95 | // } else if (b == '"' || b == '\'') { 96 | // return State.QUOTE; 97 | // } else if (isAlphaBeta(b) || isDigit(b)) { 98 | // return State.TOKEN; 99 | // } else { 100 | // err = Error.InvalidCommandException; 101 | // throw err; 102 | // } 103 | // } 104 | // 105 | // private String nextSymbolState() { 106 | // Byte b = peekByte(); 107 | // popByte(); 108 | // return new String(new byte[]{b}); 109 | // } 110 | // 111 | // private String nextTokenState() { 112 | // StringBuilder sb = new StringBuilder(); 113 | // while (true) { 114 | // Byte b = peekByte(); 115 | // if (b == null || !(isAlphaBeta(b) || isDigit(b) || b == '_')) { 116 | // if (b != null && isBlank(b)) { 117 | // popByte(); 118 | // } 119 | // return sb.toString(); 120 | // } 121 | // sb.append(new String(new byte[]{b})); 122 | // popByte(); 123 | // } 124 | // } 125 | // 126 | // private String nextQuoteState() throws Exception { 127 | // byte quote = peekByte(); 128 | // popByte(); 129 | // StringBuilder sb = new StringBuilder(); 130 | // while (true) { 131 | // Byte b = peekByte(); 132 | // if (b == null) { 133 | // err = Error.InvalidCommandException; 134 | // throw err; 135 | // } 136 | // if (b == quote) { 137 | // popByte(); 138 | // break; 139 | // } 140 | // sb.append(new String(new byte[]{b})); 141 | // popByte(); 142 | // } 143 | // return sb.toString(); 144 | // } 145 | // 146 | // private static boolean isDigit(byte b) { 147 | // return (b >= '0' && b <= '9'); 148 | // } 149 | // 150 | // private static boolean isAlphaBeta(byte b) { 151 | // return ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')); 152 | // } 153 | // 154 | // private static boolean isSymbol(byte b) { 155 | // return (b == '>' || b == '<' || b == '=' || b == '*' || 156 | // b == ',' || b == '(' || b == ')'); 157 | // } 158 | // 159 | // private static boolean isBlank(byte b) { 160 | // return (b == '\n' || b == ' ' || b == '\t'); 161 | // } 162 | //} 163 | 164 | public class Tokenizer { 165 | private enum State { 166 | INIT, IN_SYMBOL, IN_QUOTE, IN_TOKEN, END 167 | } 168 | 169 | private enum TokenType { 170 | SYMBOL, QUOTE, TOKEN 171 | } 172 | 173 | private byte[] stat; 174 | private int pos; 175 | private String currentToken; 176 | private boolean flushToken; 177 | private Exception err; 178 | private State currentState; 179 | 180 | public Tokenizer(byte[] stat) { 181 | this.stat = stat; 182 | this.pos = 0; 183 | this.currentToken = ""; 184 | this.flushToken = true; 185 | this.currentState = State.INIT; 186 | } 187 | 188 | public String peek() throws Exception { 189 | if (err != null) { 190 | throw err; 191 | } 192 | if (flushToken) { 193 | String token = null; 194 | try { 195 | token = next(); 196 | } catch (Exception e) { 197 | err = e; 198 | throw e; 199 | } 200 | currentToken = token; 201 | flushToken = false; 202 | } 203 | return currentToken; 204 | } 205 | 206 | public void pop() { 207 | flushToken = true; 208 | } 209 | 210 | public byte[] errStat() { 211 | byte[] res = new byte[stat.length + 3]; 212 | System.arraycopy(stat, 0, res, 0, pos); 213 | System.arraycopy("<< ".getBytes(), 0, res, pos, 3); 214 | System.arraycopy(stat, pos, res, pos + 3, stat.length - pos); 215 | return res; 216 | } 217 | 218 | private void popByte() { 219 | pos++; 220 | if (pos > stat.length) { 221 | pos = stat.length; 222 | } 223 | } 224 | 225 | private Byte peekByte() { 226 | if (pos == stat.length) { 227 | return null; 228 | } 229 | return stat[pos]; 230 | } 231 | 232 | private String next() throws Exception { 233 | if (err != null) { 234 | throw err; 235 | } 236 | 237 | while (true) { 238 | switch (currentState) { 239 | case INIT: 240 | currentState = nextState(); 241 | break; 242 | case IN_SYMBOL: 243 | return nextSymbolState(); 244 | case IN_QUOTE: 245 | return nextQuoteState(); 246 | case IN_TOKEN: 247 | return nextTokenState(); 248 | case END: 249 | return ""; 250 | } 251 | } 252 | } 253 | 254 | private State nextState() throws Exception { 255 | while(true) { 256 | Byte b = peekByte(); 257 | if (b == null) return State.END; 258 | if(!isBlank(b)) { 259 | break; 260 | } 261 | popByte(); 262 | } 263 | 264 | Byte b = peekByte(); 265 | if (b == null) { 266 | return State.INIT; 267 | } else if (isSymbol(b)) { 268 | return State.IN_SYMBOL; 269 | } else if (b == '"' || b == '\'') { 270 | return State.IN_QUOTE; 271 | } else if (isAlphaBeta(b) || isDigit(b)) { 272 | return State.IN_TOKEN; 273 | } else { 274 | err = Error.InvalidCommandException; 275 | throw err; 276 | } 277 | } 278 | 279 | private String nextSymbolState() { 280 | Byte b = peekByte(); 281 | popByte(); 282 | this.currentState = State.INIT; 283 | return new String(new byte[]{b}); 284 | } 285 | 286 | private String nextTokenState() { 287 | StringBuilder sb = new StringBuilder(); 288 | while (true) { 289 | Byte b = peekByte(); 290 | if (b == null || !(isAlphaBeta(b) || isDigit(b) || b == '_')) { 291 | if (b != null && isBlank(b)) { 292 | popByte(); 293 | } 294 | this.currentState = State.INIT; 295 | return sb.toString(); 296 | } 297 | sb.append(new String(new byte[]{b})); 298 | popByte(); 299 | } 300 | } 301 | 302 | private String nextQuoteState() throws Exception { 303 | byte quote = peekByte(); 304 | popByte(); 305 | StringBuilder sb = new StringBuilder(); 306 | while (true) { 307 | Byte b = peekByte(); 308 | if (b == null) { 309 | err = Error.InvalidCommandException; 310 | throw err; 311 | } 312 | if (b == quote) { 313 | popByte(); 314 | break; 315 | } 316 | sb.append(new String(new byte[]{b})); 317 | popByte(); 318 | } 319 | this.currentState = State.INIT; 320 | return sb.toString(); 321 | } 322 | 323 | private static boolean isDigit(byte b) { 324 | return (b >= '0' && b <= '9'); 325 | } 326 | 327 | static boolean isAlphaBeta(byte b) { 328 | return ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')); 329 | } 330 | 331 | private static boolean isSymbol(byte b) { 332 | return (b == '>' || b == '<' || b == '=' || b == '*' || 333 | b == ',' || b == '(' || b == ')'); 334 | } 335 | 336 | private static boolean isBlank(byte b) { 337 | return (b == '\n' || b == ' ' || b == '\t'); 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Abort.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Abort { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Begin.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Begin { 4 | public boolean isRepeatableRead; 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Commit.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Commit { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Create.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Create { 4 | public String tableName; 5 | public String[] fieldName; 6 | public String[] fieldType; 7 | public String[] index; 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Delete.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Delete { 4 | public String tableName; 5 | public Where where; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Drop.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Drop { 4 | public String tableName; 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Insert.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Insert { 4 | public String tableName; 5 | public String[] values; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Select.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Select { 4 | public String tableName; 5 | public String[] fields; 6 | public Where where; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Show.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Show { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/SingleExpression.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class SingleExpression { 4 | public String field; 5 | public String compareOp; 6 | public String value; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Update.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Update { 4 | public String tableName; 5 | public String fieldName; 6 | public String value; 7 | public Where where; 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Where.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser.statement; 2 | 3 | public class Where { 4 | public SingleExpression singleExp1; 5 | public String logicOp; 6 | public SingleExpression singleExp2; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/server/Executor.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.server; 2 | 3 | import com.northeastern.edu.simpledb.backend.parser.Parser; 4 | import com.northeastern.edu.simpledb.backend.parser.statement.*; 5 | import com.northeastern.edu.simpledb.backend.tbm.BeginRes; 6 | import com.northeastern.edu.simpledb.backend.tbm.TableManager; 7 | import com.northeastern.edu.simpledb.common.Error; 8 | 9 | public class Executor { 10 | 11 | private long xid; 12 | private TableManager tbm; 13 | 14 | public Executor(TableManager tbm) { 15 | this.tbm = tbm; 16 | this.xid = 0; 17 | } 18 | 19 | public byte[] executeTx(byte[] sql) throws Exception { 20 | System.out.println("sql = " + new String(sql)); 21 | Object statement = Parser.Parse(sql); 22 | if(statement instanceof Begin) { 23 | if(xid != 0) { 24 | throw Error.NestedTransactionException; 25 | } 26 | BeginRes r = tbm.begin((Begin)statement); 27 | xid = r.xid; 28 | return r.result; 29 | } else if(statement instanceof Commit) { 30 | if(xid == 0) { 31 | throw Error.NoTransactionException; 32 | } 33 | byte[] res = tbm.commit(xid); 34 | xid = 0; 35 | return res; 36 | } else if(statement instanceof Abort) { 37 | if(xid == 0) { 38 | throw Error.NoTransactionException; 39 | } 40 | byte[] res = tbm.abort(xid); 41 | xid = 0; 42 | return res; 43 | } else { 44 | return executeDDLAndDML(statement); 45 | } 46 | } 47 | 48 | private byte[] executeDDLAndDML(Object statement) throws Exception { 49 | boolean tmpTransaction = false; 50 | Exception e = null; 51 | if(xid == 0) { 52 | tmpTransaction = true; 53 | BeginRes r = tbm.begin(new Begin()); 54 | xid = r.xid; 55 | } 56 | try { 57 | byte[] res = null; 58 | if(statement instanceof Show) { 59 | res = tbm.show(xid); 60 | } else if(statement instanceof Create) { 61 | res = tbm.create(xid, (Create) statement); 62 | } else if(statement instanceof Select) { 63 | res = tbm.read(xid, (Select) statement); 64 | } else if(statement instanceof Insert) { 65 | res = tbm.insert(xid, (Insert) statement); 66 | } else if(statement instanceof Delete) { 67 | res = tbm.delete(xid, (Delete) statement); 68 | } else if(statement instanceof Update) { 69 | res = tbm.update(xid, (Update) statement); 70 | } 71 | return res; 72 | } catch(Exception ex) { 73 | e = ex; 74 | throw e; 75 | } finally { 76 | if(tmpTransaction) { 77 | if(e != null) { 78 | tbm.abort(xid); 79 | } else { 80 | tbm.commit(xid); 81 | } 82 | xid = 0; 83 | } 84 | } 85 | } 86 | 87 | public void close() { 88 | if (xid != 0) { 89 | System.out.println("Abnormal Abort: " + xid); 90 | tbm.abort(xid); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/server/Server.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.server; 2 | 3 | import com.northeastern.edu.simpledb.backend.tbm.TableManager; 4 | import com.northeastern.edu.simpledb.transport.Encoder; 5 | import com.northeastern.edu.simpledb.transport.Package; 6 | import com.northeastern.edu.simpledb.transport.Packager; 7 | import com.northeastern.edu.simpledb.transport.Transporter; 8 | 9 | import java.io.IOException; 10 | import java.net.InetSocketAddress; 11 | import java.net.ServerSocket; 12 | import java.net.Socket; 13 | import java.util.concurrent.ArrayBlockingQueue; 14 | import java.util.concurrent.ThreadPoolExecutor; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | public class Server { 18 | private int port; 19 | TableManager tbm; 20 | 21 | public Server(int port, TableManager tbm) { 22 | this.port = port; 23 | this.tbm = tbm; 24 | } 25 | 26 | public void start() { 27 | ServerSocket welcomeSocket = null; 28 | 29 | try { 30 | welcomeSocket = new ServerSocket(port); 31 | } catch (IOException e) { 32 | e.printStackTrace(); 33 | return ; 34 | } 35 | 36 | try { 37 | while (true) { 38 | Socket socket = welcomeSocket.accept(); 39 | 40 | System.out.println("Server listen to port: " + port); 41 | ThreadPoolExecutor tpe = new ThreadPoolExecutor(10, 20, 1L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), new ThreadPoolExecutor.CallerRunsPolicy()); 42 | 43 | Runnable worker = new HandleSocket(socket, tbm); 44 | tpe.submit(worker); 45 | } 46 | } catch (IOException e) { 47 | e.printStackTrace(); 48 | } finally { 49 | try { 50 | welcomeSocket.close(); 51 | } catch (IOException e) {} 52 | } 53 | 54 | } 55 | 56 | } 57 | class HandleSocket implements Runnable { 58 | 59 | private Socket socket; 60 | private TableManager tbm; 61 | 62 | public HandleSocket(Socket socket, TableManager tbm) { 63 | this.socket = socket; 64 | this.tbm = tbm; 65 | } 66 | 67 | @Override 68 | public void run() { 69 | InetSocketAddress address = (InetSocketAddress) socket.getRemoteSocketAddress(); 70 | System.out.println("Establish connection: " + address.getAddress().getHostAddress() + ":" + address.getPort()); 71 | Packager packager = null; 72 | try { 73 | Transporter t = new Transporter(socket); 74 | Encoder e = new Encoder(); 75 | packager = new Packager(t, e); 76 | } catch (IOException e) { 77 | e.printStackTrace(); 78 | try { 79 | socket.close(); 80 | } catch (IOException e1) { 81 | e1.printStackTrace(); 82 | } 83 | return; 84 | } 85 | Executor executor = new Executor(tbm); 86 | while (true) { 87 | Package pkg = null; 88 | try { 89 | pkg = packager.receive(); 90 | } catch (Exception e) { 91 | break; 92 | } 93 | byte[] sql = pkg.getData(); 94 | byte[] result = null; 95 | Exception e = null; 96 | try { 97 | result = executor.executeTx(sql); 98 | } catch (Exception ex) { 99 | e = ex; 100 | e.printStackTrace(); 101 | } 102 | 103 | pkg = new Package(result, e); 104 | try { 105 | packager.send(pkg); 106 | } catch (IOException ex) { 107 | ex.printStackTrace(); 108 | break; 109 | } 110 | } 111 | 112 | executor.close(); 113 | try { 114 | packager.close(); 115 | } catch (Exception e) { 116 | e.printStackTrace(); 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/AbstractTableManager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.parser.statement.*; 5 | import com.northeastern.edu.simpledb.backend.utils.Parser; 6 | import com.northeastern.edu.simpledb.backend.vm.VersionManager; 7 | 8 | public abstract class AbstractTableManager { 9 | VersionManager vm; 10 | DataManger dm; 11 | public abstract BeginRes begin(Begin begin); 12 | public abstract byte[] commit(long xid) throws Exception; 13 | public abstract byte[] abort(long xid); 14 | public abstract byte[] show(long xid); 15 | public abstract byte[] create(long xid, Create create) throws Exception; 16 | public abstract byte[] insert(long xid, Insert insert) throws Exception; 17 | public abstract byte[] read(long xid, Select select) throws Exception; 18 | public abstract byte[] update(long xid, Update update) throws Exception; 19 | public abstract byte[] delete(long xid, Delete delete) throws Exception; 20 | 21 | public static TableManager create(String path, VersionManager vm, DataManger dm) { 22 | Booter booter = Booter.create(path); 23 | booter.update(Parser.long2Byte(0)); 24 | return new TableManager(vm, dm, booter); 25 | } 26 | 27 | public static TableManager open(String path, VersionManager vm, DataManger dm) { 28 | Booter booter = Booter.open(path); 29 | return new TableManager(vm, dm ,booter); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/BeginRes.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | public class BeginRes { 4 | public long xid; 5 | public byte[] result; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/Booter.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import com.northeastern.edu.simpledb.common.Error; 5 | 6 | import java.io.File; 7 | import java.io.FileOutputStream; 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.StandardCopyOption; 11 | 12 | public class Booter { 13 | 14 | private static final String BOOTER_SUFFIX = ".bt"; 15 | private static final String BOOTER_TMP_SUFFIX = ".bt_tmp"; 16 | 17 | String path; 18 | File file; 19 | 20 | public Booter(String path, File file) { 21 | this.path = path; 22 | this.file = file; 23 | } 24 | 25 | public static Booter open(String path) { 26 | removeBadTmp(path); 27 | File tmp = new File(path + BOOTER_SUFFIX); 28 | if (!tmp.exists()) { 29 | Panic.panic(Error.FileNotExistsException); 30 | } 31 | if (!tmp.canWrite() || !tmp.canRead()) { 32 | Panic.panic(Error.FileCannotRWException); 33 | } 34 | return new Booter(path, tmp); 35 | } 36 | 37 | public static Booter create(String path) { 38 | removeBadTmp(path); 39 | File tmp = new File(path + BOOTER_SUFFIX); 40 | try { 41 | if (!tmp.createNewFile()) { 42 | Panic.panic(Error.FileExistsException); 43 | } 44 | } catch (IOException e) { 45 | Panic.panic(e); 46 | } 47 | if (!tmp.canRead() || !tmp.canWrite()) { 48 | Panic.panic(Error.FileCannotRWException); 49 | } 50 | return new Booter(path, tmp); 51 | } 52 | 53 | private static void removeBadTmp(String path) { 54 | new File(path + BOOTER_TMP_SUFFIX).delete(); 55 | } 56 | 57 | public void update(byte[] data) { 58 | File tmp = new File(path + BOOTER_TMP_SUFFIX); 59 | try { 60 | tmp.createNewFile(); 61 | } catch (Exception e) { 62 | Panic.panic(e); 63 | } 64 | if (!tmp.canRead() || !tmp.canWrite()) { 65 | Panic.panic(Error.FileCannotRWException); 66 | } 67 | try (FileOutputStream out = new FileOutputStream(tmp)) { 68 | out.write(data); 69 | out.flush(); 70 | } catch (Exception e) { 71 | Panic.panic(e); 72 | } 73 | 74 | try { 75 | Files.move(tmp.toPath(), new File(path + BOOTER_SUFFIX).toPath(), StandardCopyOption.REPLACE_EXISTING); 76 | } catch (IOException e) { 77 | Panic.panic(e); 78 | } 79 | 80 | file = new File(path + BOOTER_SUFFIX); 81 | if (!file.exists()) { 82 | Panic.panic(Error.FileNotExistsException); 83 | } 84 | if(!file.canRead() || !file.canWrite()) { 85 | Panic.panic(Error.FileCannotRWException); 86 | } 87 | } 88 | 89 | public byte[] load() { 90 | byte[] bytes = null; 91 | try { 92 | bytes = Files.readAllBytes(file.toPath()); 93 | } catch (IOException e) { 94 | Panic.panic(e); 95 | } 96 | return bytes; 97 | } 98 | 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/Field.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.backend.im.BPlusTree; 5 | import com.northeastern.edu.simpledb.backend.parser.statement.SingleExpression; 6 | import com.northeastern.edu.simpledb.backend.utils.Panic; 7 | import com.northeastern.edu.simpledb.backend.utils.ParseStringRes; 8 | import com.northeastern.edu.simpledb.backend.utils.Parser; 9 | import com.northeastern.edu.simpledb.common.Error; 10 | 11 | import java.util.Arrays; 12 | import java.util.List; 13 | 14 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID; 15 | 16 | public class Field { 17 | 18 | String fieldName; 19 | String fieldType; 20 | long index; 21 | long uid; 22 | private Table table; 23 | private BPlusTree bPlusTree; 24 | 25 | public Field(long uid, Table table) { 26 | this.uid = uid; 27 | this.table = table; 28 | } 29 | 30 | public Field(Table table, String fieldName, String fieldType, long index) { 31 | this.fieldName = fieldName; 32 | this.fieldType = fieldType; 33 | this.index = index; 34 | this.table = table; 35 | } 36 | 37 | // given table and uid, return a field 38 | public static Field loadField(Table table, long uid) { 39 | byte[] raw = null; 40 | try { 41 | raw = table.tbm.vm.read(SUPER_XID, uid); 42 | } catch (Exception e) { 43 | Panic.panic(e); 44 | } 45 | assert raw != null; 46 | return new Field(uid, table).parseSelf(raw); 47 | } 48 | 49 | public static Field createField(Table table, long xid, String fieldName, String fieldType, boolean indexed) throws Exception { 50 | typeCheck(fieldType); 51 | Field field = new Field(table, fieldName, fieldType, 0); 52 | // if this field is an index, create a B+ Tree for it 53 | if (indexed) { 54 | long index = BPlusTree.create(table.tbm.dm); 55 | BPlusTree bpt = BPlusTree.load(index, table.tbm.dm); 56 | field.index = index; 57 | field.bPlusTree = bpt; 58 | } 59 | field.persistSelf(xid); 60 | return field; 61 | } 62 | 63 | public void insert(Object key, long uid) throws Exception { 64 | long uKey = value2UKey(key); 65 | bPlusTree.insert(uKey, uid); 66 | } 67 | 68 | /** 69 | * convert object to uid key used for searching or insertion of B+ Tree 70 | * eg: where id = 8, here 8 is input object key, and it will be parsed 71 | * into int32 72 | */ 73 | private long value2UKey(Object key) { 74 | long uid = 0; 75 | switch(fieldType) { 76 | case "string": 77 | uid = Parser.str2Uid((String) key); 78 | break; 79 | case "int32": 80 | int uint = (int) key; 81 | return (long) uint; 82 | case "int64": 83 | uid = (long) key; 84 | break; 85 | } 86 | return uid; 87 | } 88 | 89 | // convert object to a byte array for storing into disk 90 | public byte[] value2Raw(Object v) { 91 | byte[] raw = null; 92 | switch(fieldType) { 93 | case "int32": 94 | raw = Parser.int2Byte((int) v); 95 | break; 96 | case "int64": 97 | raw = Parser.long2Byte((long) v); 98 | break; 99 | case "string": 100 | raw = Parser.string2Byte((String) v); 101 | break; 102 | } 103 | return raw; 104 | } 105 | 106 | // convert single expression into FieldCalRes 107 | public FieldCalRes calExp(SingleExpression exp) { 108 | Object v; 109 | FieldCalRes fieldCalRes = new FieldCalRes(); 110 | switch (exp.compareOp) { 111 | case "<": 112 | fieldCalRes.left = 0; 113 | v = string2Value(exp.value); 114 | long right = value2UKey(v); 115 | fieldCalRes.right = right; 116 | if (fieldCalRes.right > 0) { 117 | fieldCalRes.right --; 118 | } 119 | break; 120 | case "=": 121 | v = string2Value(exp.value); 122 | fieldCalRes.left = value2UKey(v); 123 | fieldCalRes.right = fieldCalRes.left; 124 | break; 125 | case ">": 126 | fieldCalRes.right = Long.MAX_VALUE; 127 | v = string2Value(exp.value); 128 | fieldCalRes.left = value2UKey(v); 129 | break; 130 | } 131 | return fieldCalRes; 132 | } 133 | 134 | 135 | private Field parseSelf(byte[] raw) { 136 | int position = 0; 137 | ParseStringRes parseStringRes = null; 138 | parseStringRes = Parser.parseString(raw); 139 | position += parseStringRes.next; 140 | fieldName = parseStringRes.str; 141 | parseStringRes = Parser.parseString(Arrays.copyOfRange(raw, position, raw.length)); 142 | fieldType = parseStringRes.str; 143 | position += parseStringRes.next; 144 | index = Parser.parseLong(Arrays.copyOfRange(raw, position, position + Long.BYTES)); 145 | if (index != 0) { 146 | try { 147 | BPlusTree.load(index, table.tbm.dm); 148 | } catch (Exception e) { 149 | Panic.panic(e); 150 | } 151 | } 152 | return this; 153 | } 154 | 155 | private void persistSelf(long xid) throws Exception { 156 | byte[] nameRaw = Parser.string2Byte(fieldName); 157 | byte[] fieldRaw = Parser.string2Byte(fieldType); 158 | byte[] indexRaw = Parser.long2Byte(index); 159 | this.uid = table.tbm.vm.insert(xid, Bytes.concat(nameRaw, fieldRaw, indexRaw)); 160 | } 161 | 162 | public Object string2Value(String str) { 163 | switch(fieldType) { 164 | case "int32": 165 | return Integer.parseInt(str); 166 | case "int64": 167 | return Long.parseLong(str); 168 | case "string": 169 | return str; 170 | } 171 | return null; 172 | } 173 | 174 | private static void typeCheck(String fieldType) throws Exception { 175 | if(!"int32".equals(fieldType) && !"int64".equals(fieldType) && !"string".equals(fieldType)) { 176 | throw Error.InvalidFieldException; 177 | } 178 | } 179 | 180 | public boolean isIndexed() { 181 | return index != 0; 182 | } 183 | 184 | public List search(long l, long r) throws Exception { 185 | return bPlusTree.searchRange(l, r); 186 | } 187 | 188 | public String printValue(Object v) { 189 | String str = null; 190 | switch(fieldType) { 191 | case "int32": 192 | str = String.valueOf((int) v); 193 | break; 194 | case "int64": 195 | str = String.valueOf((long) v); 196 | break; 197 | case "string": 198 | str = (String) v; 199 | break; 200 | } 201 | return str; 202 | } 203 | 204 | class ParseValueRes { 205 | Object v; 206 | int shift; 207 | } 208 | 209 | public ParseValueRes parseValue(byte[] raw) { 210 | ParseValueRes parseValueRes = new ParseValueRes(); 211 | switch (fieldType) { 212 | case "int32": 213 | parseValueRes.v = Parser.parseInt(raw); 214 | parseValueRes.shift = Integer.BYTES; 215 | break; 216 | case "int64": 217 | parseValueRes.v = Parser.parseLong(raw); 218 | parseValueRes.shift = Long.BYTES; 219 | break; 220 | case "string": 221 | ParseStringRes parseStringRes = Parser.parseString(raw); 222 | parseValueRes.v = parseStringRes.str; 223 | parseValueRes.shift = parseStringRes.next; 224 | break; 225 | } 226 | return parseValueRes; 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/FieldCalRes.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | public class FieldCalRes { 4 | public long left; 5 | public long right; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tbm/TableManager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tbm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.parser.statement.*; 5 | import com.northeastern.edu.simpledb.backend.utils.Parser; 6 | import com.northeastern.edu.simpledb.backend.vm.VersionManager; 7 | import com.northeastern.edu.simpledb.common.Error; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.concurrent.ConcurrentHashMap; 13 | import java.util.concurrent.locks.Lock; 14 | import java.util.concurrent.locks.ReentrantLock; 15 | 16 | public class TableManager extends AbstractTableManager{ 17 | 18 | private Booter booter; 19 | private Map tableCache; 20 | private Map> xidTableCache; 21 | private Lock lock; 22 | 23 | public TableManager(VersionManager vm, DataManger dm, Booter booter) { 24 | this.booter = booter; 25 | this.vm = vm; 26 | this.dm = dm; 27 | this.tableCache = new ConcurrentHashMap<>(); 28 | this.xidTableCache = new ConcurrentHashMap<>(); 29 | lock = new ReentrantLock(); 30 | loadTables(); 31 | } 32 | 33 | private void loadTables() { 34 | long uid = firstTableUid(); 35 | while (uid != 0) { 36 | Table table = Table.loadTable(this, uid); 37 | uid = table.nextUid; 38 | tableCache.put(table.name, table); 39 | } 40 | } 41 | 42 | private long firstTableUid() { 43 | byte[] raw = booter.load(); 44 | return Parser.parseLong(raw); 45 | } 46 | 47 | private void updateFirstTableUid(long uid) { 48 | byte[] raw = Parser.long2Byte(uid); 49 | booter.update(raw); 50 | } 51 | 52 | @Override 53 | public BeginRes begin(Begin begin) { 54 | BeginRes beginRes = new BeginRes(); 55 | int isolationLevel = begin.isRepeatableRead ? 1 : 0; 56 | beginRes.xid = vm.begin(isolationLevel); 57 | beginRes.result = "begin".getBytes(); 58 | return beginRes; 59 | } 60 | 61 | @Override 62 | public byte[] commit(long xid) throws Exception { 63 | vm.commit(xid); 64 | return "commit".getBytes(); 65 | } 66 | 67 | @Override 68 | public byte[] abort(long xid) { 69 | vm.abort(xid); 70 | return "abort".getBytes(); 71 | } 72 | 73 | @Override 74 | public byte[] show(long xid) { 75 | lock.lock(); 76 | try { 77 | StringBuilder sb = new StringBuilder(); 78 | for (Table tb : tableCache.values()) { 79 | sb.append(tb.toString()).append("\n"); 80 | } 81 | List t = xidTableCache.get(xid); 82 | if(t == null) { 83 | return "\n".getBytes(); 84 | } 85 | for (Table tb : t) { 86 | sb.append(tb.toString()).append("\n"); 87 | } 88 | return sb.toString().getBytes(); 89 | } finally { 90 | lock.unlock(); 91 | } 92 | } 93 | 94 | @Override 95 | public byte[] create(long xid, Create create) throws Exception { 96 | lock.lock(); 97 | try { 98 | if (tableCache.containsKey(create.tableName)) throw Error.DuplicatedTableException; 99 | Table table = Table.createTable(this, firstTableUid(), xid, create); // How does it work? 100 | updateFirstTableUid(table.uid); 101 | tableCache.put(create.tableName, table); 102 | xidTableCache.computeIfAbsent(xid, k -> new ArrayList<>()).add(table); 103 | return ("create " + create.tableName).getBytes(); 104 | } finally { 105 | lock.unlock(); 106 | } 107 | } 108 | 109 | @Override 110 | public byte[] insert(long xid, Insert insert) throws Exception { 111 | Table table = tableCache.get(insert.tableName); 112 | if (table == null) throw Error.TableNotFoundException; 113 | table.insert(xid, insert); 114 | return "insert".getBytes(); 115 | } 116 | 117 | @Override 118 | public byte[] read(long xid, Select read) throws Exception { 119 | Table table = tableCache.get(read.tableName); 120 | if (table == null) throw Error.TableNotFoundException; 121 | return table.read(xid, read).getBytes(); 122 | } 123 | 124 | @Override 125 | public byte[] update(long xid, Update update) throws Exception { 126 | Table table = tableCache.get(update.tableName); 127 | if(table == null) throw Error.TableNotFoundException; 128 | int count = table.update(xid, update); 129 | return ("update " + count).getBytes(); 130 | } 131 | 132 | @Override 133 | public byte[] delete(long xid, Delete delete) throws Exception { 134 | Table table = tableCache.get(delete.tableName); 135 | if(table == null) throw Error.TableNotFoundException; 136 | int count = table.delete(xid, delete); 137 | return ("delete " + count).getBytes(); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tm/AbstractTransactionManager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tm; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import com.northeastern.edu.simpledb.common.Error; 5 | 6 | import java.io.File; 7 | import java.io.FileNotFoundException; 8 | import java.io.IOException; 9 | import java.io.RandomAccessFile; 10 | import java.nio.ByteBuffer; 11 | import java.nio.channels.FileChannel; 12 | 13 | public abstract class AbstractTransactionManager { 14 | abstract long begin(); 15 | abstract void commit(long xid); 16 | abstract void abort(long xid); 17 | abstract boolean isActive(long xid); 18 | abstract boolean isCommitted(long xid); 19 | abstract boolean isAborted(long xid); 20 | abstract void close(); 21 | 22 | public static TransactionManager create(String path) { 23 | File f = new File(path + TransactionManager.XID_SUFFIX); 24 | try { 25 | if (!f.createNewFile()) { 26 | Panic.panic(Error.BadXIDFileException); 27 | } 28 | } catch (IOException e) { 29 | Panic.panic(e); 30 | } 31 | 32 | RandomAccessFile raf = null; 33 | FileChannel fc = null; 34 | try { 35 | raf = new RandomAccessFile(f, "rw"); 36 | fc = raf.getChannel(); 37 | } catch (IOException e) { 38 | Panic.panic(e); 39 | } 40 | 41 | ByteBuffer buf = ByteBuffer.allocate(TransactionManager.XID_HEADER_LENGTH); 42 | try { 43 | fc.position(0); 44 | fc.write(buf); 45 | } catch (IOException e) { 46 | Panic.panic(e); 47 | } 48 | 49 | return new TransactionManager(raf, fc); 50 | } 51 | 52 | public static TransactionManager open(String path) { 53 | File f = new File(path + TransactionManager.XID_SUFFIX); 54 | if(!f.exists()) { 55 | Panic.panic(Error.FileNotExistsException); 56 | } 57 | if(!f.canRead() || !f.canWrite()) { 58 | Panic.panic(Error.FileCannotRWException); 59 | } 60 | FileChannel fc = null; 61 | RandomAccessFile raf = null; 62 | try { 63 | raf = new RandomAccessFile(f, "rw"); 64 | fc = raf.getChannel(); 65 | } catch (FileNotFoundException e) { 66 | Panic.panic(e); 67 | } 68 | 69 | return new TransactionManager(raf, fc); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/tm/TransactionManager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tm; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import com.northeastern.edu.simpledb.backend.utils.Parser; 5 | import com.northeastern.edu.simpledb.common.Error; 6 | 7 | import java.io.IOException; 8 | import java.io.RandomAccessFile; 9 | import java.nio.ByteBuffer; 10 | import java.nio.channels.FileChannel; 11 | import java.util.concurrent.locks.Lock; 12 | import java.util.concurrent.locks.ReentrantLock; 13 | 14 | 15 | /* 16 | 17 | */ 18 | public class TransactionManager extends AbstractTransactionManager{ 19 | 20 | // the length of header of xid file, unit is byte 21 | static final int XID_HEADER_LENGTH = 8; 22 | 23 | // the size of each xid, unit is byte 24 | private static final int XID_FIELD_SIZE = 1; 25 | 26 | // three states of transaction 27 | private static final byte FIELD_TRAN_ACTIVE = 0; 28 | private static final byte FIELD_TRAN_COMMITTED = 1; 29 | private static final byte FIELD_TRAN_ABORTED = 2; 30 | 31 | // super transaction whose state is always committed 32 | public static final long SUPER_XID = 0; 33 | 34 | // standard suffix of xid file 35 | public static final String XID_SUFFIX = ".xid"; 36 | 37 | private RandomAccessFile file; 38 | private FileChannel fc; 39 | private long xidCounter; 40 | private Lock counterLock; 41 | 42 | public TransactionManager(RandomAccessFile raf, FileChannel fc) { 43 | this.file = raf; 44 | this.fc = fc; 45 | this.counterLock = new ReentrantLock(); 46 | checkXIDCounter(); 47 | } 48 | 49 | /** 50 | * check if the xid file is valid 51 | * read the xidcounter in XID_FILE_HEADER, calculate 52 | * the theoretical length of the file based on it, and 53 | * compare it with the actual length 54 | */ 55 | private void checkXIDCounter() { 56 | long fileLength = 0; 57 | try { 58 | fileLength = file.length(); 59 | } catch (IOException e) { 60 | Panic.panic(Error.BadXIDFileException); 61 | } 62 | if (fileLength < XID_HEADER_LENGTH) { 63 | Panic.panic(Error.BadXIDFileException); 64 | } 65 | 66 | ByteBuffer buf = ByteBuffer.allocate(XID_HEADER_LENGTH); 67 | try { 68 | fc.position(0); 69 | fc.read(buf); 70 | } catch (IOException e) { 71 | Panic.panic(e); 72 | } 73 | this.xidCounter = Parser.parseLong(buf.array()); 74 | long end = getXidPosition(this.xidCounter + 1); 75 | if (end != fileLength) { 76 | Panic.panic(Error.BadXIDFileException); 77 | } 78 | } 79 | 80 | /** 81 | * calculate its position in xid file based on input `xid` 82 | * `xid` means it is the normal xid transaction starting from 1 83 | * the xid of super transaction is 0 84 | */ 85 | private long getXidPosition(long xid) { 86 | if (xid == 0) { 87 | throw new RuntimeException("position of xid can't be zero"); 88 | } 89 | return (xid - 1) * XID_FIELD_SIZE + XID_HEADER_LENGTH; 90 | } 91 | 92 | // update state by xid 93 | private void updateXID(long xid, byte state) { 94 | long offset = getXidPosition(xid); 95 | byte[] tmp = new byte[XID_FIELD_SIZE]; 96 | tmp[0] = state; 97 | try { 98 | fc.position(offset); 99 | fc.write(ByteBuffer.wrap(tmp)); 100 | } catch (IOException e) { 101 | Panic.panic(e); 102 | } 103 | 104 | flushToDisk(); 105 | } 106 | 107 | // increase one to xid counter and update header of xid file 108 | private void incrXIDCounter() { 109 | xidCounter++; 110 | ByteBuffer buf = ByteBuffer.wrap(Parser.long2Byte(xidCounter)); 111 | try { 112 | fc.position(0); 113 | fc.write(buf); 114 | } catch (IOException e) { 115 | Panic.panic(e); 116 | } 117 | 118 | flushToDisk(); 119 | } 120 | 121 | private void flushToDisk() { 122 | try { 123 | fc.force(false); 124 | } catch (IOException e) { 125 | Panic.panic(e); 126 | } 127 | } 128 | 129 | // start a new transaction 130 | @Override 131 | public long begin() { 132 | counterLock.lock(); 133 | try { 134 | long xid = xidCounter + 1; 135 | updateXID(xid, FIELD_TRAN_ACTIVE); 136 | incrXIDCounter(); 137 | return xid; 138 | } finally { 139 | counterLock.unlock(); 140 | } 141 | } 142 | 143 | // commit a transaction 144 | @Override 145 | public void commit(long xid) { 146 | updateXID(xid, FIELD_TRAN_COMMITTED); 147 | } 148 | 149 | // abort a transaction 150 | @Override 151 | public void abort(long xid) { 152 | updateXID(xid, FIELD_TRAN_ABORTED); 153 | } 154 | 155 | // check if xid transaction is in input `state` 156 | private boolean checkXID(long xid, byte state) { 157 | long offset = getXidPosition(xid); 158 | ByteBuffer buf = ByteBuffer.allocate(XID_FIELD_SIZE); 159 | try { 160 | fc.position(offset); 161 | fc.read(buf); 162 | } catch (IOException e) { 163 | Panic.panic(e); 164 | } 165 | return buf.array()[0] == state; 166 | } 167 | 168 | @Override 169 | public boolean isActive(long xid) { 170 | if(xid == SUPER_XID) return false; 171 | return checkXID(xid, FIELD_TRAN_ACTIVE); 172 | } 173 | 174 | @Override 175 | public boolean isCommitted(long xid) { 176 | if(xid == SUPER_XID) return true; 177 | return checkXID(xid, FIELD_TRAN_COMMITTED); 178 | } 179 | 180 | @Override 181 | public boolean isAborted(long xid) { 182 | if(xid == SUPER_XID) return false; 183 | return checkXID(xid, FIELD_TRAN_ABORTED); 184 | } 185 | 186 | @Override 187 | public void close() { 188 | try { 189 | fc.close(); 190 | file.close(); 191 | } catch (IOException e) { 192 | Panic.panic(e); 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/utils/Panic.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.utils; 2 | 3 | public class Panic { 4 | public static void panic(Exception err) { 5 | err.printStackTrace(); 6 | System.exit(1); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/utils/ParseStringRes.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.utils; 2 | 3 | public class ParseStringRes { 4 | public String str; 5 | public int next; 6 | 7 | public ParseStringRes(String str, int next) { 8 | this.str = str; 9 | this.next = next; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/utils/Parser.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.utils; 2 | 3 | import com.google.common.primitives.Bytes; 4 | 5 | import java.nio.ByteBuffer; 6 | import java.nio.charset.StandardCharsets; 7 | import java.util.Arrays; 8 | 9 | public class Parser { 10 | 11 | public static long parseLong(byte[] bytes) { 12 | ByteBuffer buf = ByteBuffer.wrap(bytes, 0, Long.BYTES); 13 | return buf.getLong(); 14 | } 15 | 16 | public static byte[] long2Byte(long value) { 17 | return ByteBuffer.allocate(Long.BYTES).putLong(value).array(); 18 | } 19 | 20 | public static short parseShort(byte[] bytes) { 21 | ByteBuffer buf = ByteBuffer.wrap(bytes, 0, Short.BYTES); 22 | return buf.getShort(); 23 | } 24 | 25 | public static byte[] short2Byte(short value) { 26 | return ByteBuffer.allocate(Short.BYTES).putShort(value).array(); 27 | } 28 | 29 | public static byte[] int2Byte(int value) { 30 | return ByteBuffer.allocate(Integer.BYTES).putInt(value).array(); 31 | } 32 | 33 | public static int parseInt(byte[] raw) { 34 | return ByteBuffer.wrap(raw, 0, Integer.BYTES).getInt(); 35 | } 36 | 37 | public static ParseStringRes parseString(byte[] raw) { 38 | int length = parseInt(Arrays.copyOf(raw, Integer.BYTES)); 39 | String str = new String(Arrays.copyOfRange(raw, Integer.BYTES, Integer.BYTES + length)); 40 | return new ParseStringRes(str, length + Integer.BYTES); 41 | } 42 | 43 | public static byte[] string2Byte(String str) { 44 | byte[] length = int2Byte(str.length()); 45 | return Bytes.concat(length, str.getBytes(StandardCharsets.UTF_8)); 46 | } 47 | 48 | public static long str2Uid(String key) { 49 | long seed = 13331; 50 | long res = 0; 51 | for(byte b : key.getBytes()) { 52 | res = res * seed + (long)b; 53 | } 54 | return res; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/utils/RandomUtil.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.utils; 2 | 3 | import java.security.SecureRandom; 4 | 5 | public class RandomUtil { 6 | 7 | public static byte[] randomBytes(int length) { 8 | SecureRandom random = new SecureRandom(); 9 | byte[] bytes = new byte[length]; 10 | random.nextBytes(bytes); 11 | return bytes; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/utils/Types.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.utils; 2 | 3 | public class Types { 4 | public static long addressToUid(int pageNumber, short offset) { 5 | long u0 = pageNumber; 6 | long u1 = offset; 7 | return u0 << 32 | u1; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/Entry.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.backend.common.SubArray; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.utils.Parser; 7 | 8 | import java.util.Arrays; 9 | 10 | /** 11 | * VM provide Entry to the upper layer 12 | * [XMIN][XMAX][data] 13 | */ 14 | public class Entry { 15 | 16 | private static final int OF_XMIN = 0; 17 | private static final int OF_XMAX = OF_XMIN + 8; 18 | private static final int OF_DATA = OF_XMAX + 8; 19 | 20 | private long uid; 21 | private DataItem dataItem; 22 | private VersionManager vm; 23 | private static Entry newEntry(VersionManager vm, DataItem di, long uid) { 24 | Entry entry = new Entry(); 25 | entry.uid = uid; 26 | entry.dataItem = di; 27 | entry.vm = vm; 28 | return entry; 29 | } 30 | 31 | public static Entry loadEntry(VersionManager vm, long uid) throws Exception { 32 | DataItem di = vm.dm.read(uid); 33 | return newEntry(vm, di, uid); 34 | } 35 | 36 | public static byte[] wrapEntryRaw(long xid, byte[] data) { 37 | byte[] xmin = Parser.long2Byte(xid); 38 | byte[] xmax = new byte[8]; 39 | return Bytes.concat(xmin, xmax, data); 40 | } 41 | 42 | public void release() { 43 | vm.releaseEntry(this); 44 | } 45 | 46 | public byte[] data() { 47 | dataItem.rLock(); 48 | try { 49 | SubArray subArray = dataItem.data(); 50 | byte[] data = new byte[subArray.end - (subArray.start + OF_DATA)]; 51 | System.arraycopy(subArray.raw, subArray.start + OF_DATA, data, 0, data.length); 52 | return data; 53 | } finally { 54 | dataItem.rUnLock(); 55 | } 56 | } 57 | 58 | public long getXmin() { 59 | dataItem.rLock(); 60 | try { 61 | SubArray subArray = dataItem.data(); 62 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start + OF_XMIN, subArray.start + OF_XMAX)); 63 | } finally { 64 | dataItem.rUnLock(); 65 | } 66 | } 67 | 68 | public long getXmax() { 69 | dataItem.rLock(); 70 | try { 71 | SubArray subArray = dataItem.data(); 72 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start + OF_XMAX, subArray.start + OF_DATA)); 73 | } finally { 74 | dataItem.rUnLock(); 75 | } 76 | } 77 | 78 | public void setXmax(long xid) { 79 | dataItem.before(); 80 | try { 81 | SubArray subArray = dataItem.data(); 82 | System.arraycopy(Parser.long2Byte(xid), 0, subArray.raw, subArray.start + OF_XMAX, 8); 83 | } finally { 84 | dataItem.after(xid); 85 | } 86 | } 87 | 88 | public long getUid() { 89 | return uid; 90 | } 91 | 92 | public void remove() { 93 | dataItem.release(); 94 | } 95 | 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/LockTable.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import java.util.*; 4 | import java.util.concurrent.locks.Lock; 5 | import java.util.concurrent.locks.ReentrantLock; 6 | 7 | import com.northeastern.edu.simpledb.common.Error; 8 | import com.northeastern.edu.simpledb.common.Error.*; 9 | 10 | public class LockTable { 11 | 12 | /** 13 | * what uids that xid is holding 14 | * format: {xid: [uid0, uid1, uid2...]} 15 | */ 16 | private Map> x2u; 17 | 18 | /** 19 | * what xid is holding the uid 20 | * format: {uid: xid} 21 | */ 22 | private Map u2x; 23 | 24 | /** 25 | * all of xid are waiting uid 26 | * format: {uid: [xid0, xid1, xid2...]} 27 | */ 28 | private Map> wait; 29 | 30 | /** 31 | * the lock of xid is waiting resource 32 | * format: {xid: Lock} 33 | */ 34 | private Map waitLock; 35 | 36 | /** 37 | * what uid that xid is waiting for 38 | * format: {xid : uid} 39 | */ 40 | private Map waitU; 41 | 42 | private Lock lock; 43 | 44 | public LockTable() { 45 | x2u = new HashMap<>(); 46 | u2x = new HashMap<>(); 47 | wait = new HashMap<>(); 48 | waitLock = new HashMap<>(); 49 | waitU = new HashMap<>(); 50 | lock = new ReentrantLock(); 51 | } 52 | 53 | // add a relationship which is xid is waiting uid 54 | public Lock add(long xid, long uid) throws Exception{ 55 | lock.lock(); 56 | try { 57 | // xid is holding uid 58 | if (isInList(x2u, xid, uid)) return null; 59 | 60 | // if no one is holding uid, add the relationship directly (acquired resource) 61 | if (!u2x.containsKey(uid)) { 62 | u2x.put(uid, xid); 63 | putIntoList(x2u, xid, uid); 64 | return null; 65 | } 66 | 67 | // other is holding uid 68 | // try to wait 69 | waitU.put(xid, uid); 70 | putIntoList(wait, uid, xid); 71 | 72 | // check deadlock 73 | if (hasDeadLock()) { 74 | waitU.remove(xid); 75 | removeFromList(wait, uid, xid); 76 | throw Error.DeadLockException; 77 | } 78 | 79 | // no deadlock, start waiting 80 | Lock l = new ReentrantLock(); 81 | l.lock(); 82 | waitLock.put(xid, l); // store lock for the releasing later 83 | return l; 84 | } finally { 85 | lock.unlock(); 86 | } 87 | } 88 | 89 | // remove a relationship between xid and uid 90 | public void remove(long xid) { 91 | lock.lock(); 92 | try { 93 | List uids = x2u.get(xid); 94 | if (uids != null) { 95 | while (uids.size() > 0) { 96 | Long uid = uids.remove(0); 97 | selectNewXID(uid); // select the next xid for uid from candidates 98 | } 99 | } 100 | } finally { 101 | lock.unlock(); 102 | } 103 | } 104 | 105 | // select the next xid for uid from candidates 106 | private void selectNewXID(Long uid) { 107 | u2x.remove(uid); // remove the relationship which is uid is acquired by xid 108 | List xids = wait.get(uid); // get candidates queue 109 | if (xids == null) return ; 110 | assert xids.size() > 0; 111 | 112 | while (xids.size() > 0) { 113 | Long xid = xids.remove(0); 114 | if (!waitLock.containsKey(xid)) { // no lock found for the xid 115 | continue; 116 | } else { // assign uid this xid 117 | u2x.put(uid, xid); 118 | putIntoList(x2u, xid, uid); 119 | Lock l = waitLock.remove(xid); 120 | waitU.remove(xid); 121 | l.unlock(); 122 | } 123 | } 124 | } 125 | 126 | private void removeFromList(Map> wait, long uid, long xid) { 127 | List list = wait.getOrDefault(uid, List.of()); 128 | Iterator iterator = list.iterator(); 129 | while (iterator.hasNext()) { 130 | Long e; 131 | if ((e = iterator.next()) == xid) { 132 | iterator.remove(); 133 | break; 134 | } 135 | } 136 | if (list.size() == 0) wait.remove(uid); 137 | } 138 | 139 | private void putIntoList(Map> map, long key, long value) { 140 | map.computeIfAbsent(key, k -> new ArrayList<>()).add(0, value); 141 | } 142 | 143 | private boolean isInList(Map> x2u, long xid, long uid) { 144 | return x2u.getOrDefault(xid, List.of()).stream().anyMatch(e -> e == uid); 145 | } 146 | 147 | 148 | private Map xidStamp; 149 | private int stamp; 150 | 151 | private boolean hasDeadLock() { 152 | xidStamp = new HashMap<>(); 153 | stamp = 1; 154 | // iterate x2u 155 | for (long xid : x2u.keySet()) { 156 | Integer s = xidStamp.get(xid); 157 | if (s != null && s > 0) continue; 158 | stamp++; 159 | if(dfs(xid)) { 160 | return true; 161 | } 162 | } 163 | return false; 164 | } 165 | 166 | /** 167 | * see what uid that the xid is waiting for, and 168 | * check if another xid holding the uid is waiting 169 | * the uid acquired by the xid 170 | */ 171 | private boolean dfs(long xid) { 172 | Integer stp = xidStamp.get(xid); 173 | if(stp != null && stp == stamp) { 174 | return true; 175 | } 176 | if(stp != null && stp < stamp) { 177 | return false; 178 | } 179 | xidStamp.put(xid, stamp); 180 | 181 | Long uid = waitU.get(xid); // what uid that this xid is waiting for 182 | if(uid == null) return false; 183 | Long x = u2x.get(uid); // what xid is holding this uid 184 | assert x != null; 185 | return dfs(x); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/Transaction.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID; 7 | 8 | public class Transaction { 9 | public long xid; 10 | public int level; 11 | public Map snapshot; 12 | public Exception err; 13 | public boolean autoAborted; 14 | 15 | public static Transaction newTransaction(long xid, int level, Map active) { 16 | Transaction transaction = new Transaction(); 17 | transaction.xid = xid; 18 | transaction.level = level; 19 | if (level != 0) { 20 | transaction.snapshot = new HashMap<>(); 21 | for (Long x : active.keySet()) { 22 | transaction.snapshot.put(x, true); 23 | } 24 | } 25 | return transaction; 26 | } 27 | 28 | public boolean isInSnapShot(long xid) { 29 | if (xid == SUPER_XID) return false; 30 | return snapshot.containsKey(xid); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/VersionManager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache; 4 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 5 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 6 | import com.northeastern.edu.simpledb.backend.utils.Panic; 7 | import com.northeastern.edu.simpledb.common.Error; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | import java.util.concurrent.locks.Lock; 13 | import java.util.concurrent.locks.ReentrantLock; 14 | 15 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID; 16 | import static com.northeastern.edu.simpledb.common.Error.NullEntryException; 17 | 18 | public class VersionManager extends AbstractCache implements VersionManagerHandler { 19 | 20 | TransactionManager tm; 21 | DataManger dm; 22 | Map activeTransaction; 23 | Lock lock; 24 | LockTable lt; 25 | 26 | public VersionManager(TransactionManager tm, DataManger dm) { 27 | super(0); 28 | this.tm = tm; 29 | this.dm = dm; 30 | activeTransaction = new ConcurrentHashMap<>(); 31 | activeTransaction.put(SUPER_XID, Transaction.newTransaction(SUPER_XID, 0, null)); 32 | this.lock = new ReentrantLock(); 33 | this.lt = new LockTable(); 34 | } 35 | 36 | @Override 37 | public byte[] read(long xid, long uid) throws Exception { 38 | Transaction transaction = activeTransaction.get(xid); 39 | if (transaction == null) return null; 40 | if (transaction.err != null) throw transaction.err; 41 | 42 | Entry entry = null; 43 | try { 44 | entry = super.get(uid); 45 | } catch (Exception e) { 46 | if (e == NullEntryException) { 47 | return null; 48 | } else throw e; 49 | } 50 | 51 | try { 52 | if (Visibility.isVisible(tm, transaction, entry)) { 53 | return entry.data(); 54 | } else return null; 55 | } finally { 56 | entry.release(); 57 | } 58 | } 59 | 60 | @Override 61 | public long insert(long xid, byte[] data) throws Exception { 62 | Transaction transaction = activeTransaction.get(xid); 63 | if (transaction == null) throw Error.InvalidCommandException; 64 | if (transaction.err != null) throw transaction.err; 65 | 66 | byte[] raw = Entry.wrapEntryRaw(xid, data); 67 | return dm.insert(xid, raw); 68 | } 69 | 70 | @Override 71 | public boolean delete(long xid, long uid) throws Exception { 72 | Transaction transaction = activeTransaction.get(xid); 73 | if (transaction == null) throw Error.InvalidCommandException; 74 | if (transaction.err != null) throw transaction.err; 75 | 76 | 77 | Entry entry = null; 78 | try { 79 | entry = super.get(uid); 80 | } catch (Exception e) { 81 | if (e == NullEntryException) { 82 | return false; 83 | } else throw e; 84 | } 85 | 86 | try { 87 | if (!Visibility.isVisible(tm, transaction, entry)) { 88 | return false; 89 | } 90 | 91 | Lock l = null; 92 | try { 93 | l = lt.add(xid, uid); // try to delete it 94 | } catch (Exception e) { 95 | transaction.err = Error.ConcurrentUpdateException; 96 | internAbort(xid, true); 97 | transaction.autoAborted = true; 98 | throw transaction.err; 99 | } 100 | 101 | if (l != null) { 102 | l.lock(); 103 | l.unlock(); 104 | } 105 | 106 | if (entry.getXmax() == xid) { // has been deleted 107 | return false; 108 | } 109 | 110 | if (Visibility.isVersionSkip(tm, transaction, entry)) { // version skip, do rollback 111 | transaction.err = Error.ConcurrentUpdateException; 112 | internAbort(xid, true); 113 | transaction.autoAborted = true; 114 | throw transaction.err; 115 | } 116 | 117 | entry.setXmax(xid); // signal entry as deleted 118 | return true; 119 | 120 | } finally { 121 | entry.release(); 122 | } 123 | } 124 | 125 | /** 126 | * abort delete when exception happens 127 | * @param autoAborted: if it happens by accident, autoAborted is true 128 | */ 129 | private void internAbort(long xid, boolean autoAborted) { 130 | lock.lock(); 131 | Transaction transaction = activeTransaction.get(xid); 132 | if (!autoAborted) activeTransaction.remove(xid); 133 | lock.unlock(); 134 | 135 | if (!transaction.autoAborted) { 136 | lt.remove(xid); 137 | tm.abort(xid); 138 | } 139 | } 140 | 141 | /** 142 | * start a transaction 143 | * when new a transaction, do a snapshot of active transaction 144 | * for the validation of visibility 145 | */ 146 | @Override 147 | public long begin(int level) { 148 | lock.lock(); 149 | try { 150 | long xid = tm.begin(); 151 | Transaction transaction = Transaction.newTransaction(xid, level, activeTransaction); 152 | activeTransaction.put(xid, transaction); 153 | return xid; 154 | } finally { 155 | lock.unlock(); 156 | } 157 | } 158 | 159 | @Override 160 | public void commit(long xid) throws Exception { 161 | Transaction transaction = activeTransaction.get(xid); 162 | 163 | try { 164 | if (transaction.err != null) throw transaction.err; 165 | } catch (NullPointerException e) { 166 | System.out.println(xid); 167 | System.out.println(activeTransaction.keySet()); 168 | Panic.panic(e); 169 | } 170 | 171 | activeTransaction.remove(xid); 172 | 173 | lt.remove(xid); 174 | tm.commit(xid); 175 | } 176 | 177 | @Override 178 | public void abort(long xid) { 179 | internAbort(xid, false); 180 | } 181 | 182 | @Override 183 | protected Entry getForCache(long uid) throws Exception { 184 | Entry entry = Entry.loadEntry(this, uid); 185 | if (entry == null) throw NullEntryException; 186 | return entry; 187 | } 188 | 189 | @Override 190 | protected void releaseForCache(Entry entry) { 191 | entry.remove(); 192 | } 193 | 194 | public void releaseEntry(Entry entry) { 195 | super.release(entry.getUid()); 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/VersionManagerHandler.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 5 | 6 | public interface VersionManagerHandler { 7 | byte[] read(long xid, long uid) throws Exception; 8 | long insert(long xid, byte[] data) throws Exception; 9 | boolean delete(long xid, long uid) throws Exception; 10 | 11 | long begin(int level); 12 | void commit(long xid) throws Exception; 13 | void abort(long xid); 14 | 15 | static VersionManagerHandler newVersionManager(TransactionManager tm, DataManger dm) { 16 | return new VersionManager(tm, dm); 17 | } 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/backend/vm/Visibility.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 4 | 5 | public class Visibility { 6 | 7 | /** 8 | * version skip happens when e has been updated by t1 from v0 9 | * to v1, but t1 can't be seen by t2, and t2 try to update e, 10 | * the version of it will transfer from v0 to v2, skipping v1 11 | */ 12 | public static boolean isVersionSkip(TransactionManager tm, Transaction t, Entry e) { 13 | long xmax = e.getXmax(); 14 | if (t.level == 0) return false; 15 | else return tm.isCommitted(xmax) && (xmax > t.xid || t.isInSnapShot(xmax)); 16 | } 17 | 18 | // determine if record(e) is visible to transaction(t) based on isolation level 19 | protected static boolean isVisible(TransactionManager tm, Transaction t, Entry e) { 20 | if(t.level == 0) return readCommitted(tm, t, e); 21 | else return repeatableRead(tm, t, e); 22 | } 23 | 24 | private static boolean readCommitted(TransactionManager tm, Transaction t, Entry e) { 25 | long xid = t.xid; 26 | long xmin = e.getXmin(); 27 | long xmax = e.getXmax(); 28 | if (xmin == xid && xmax == 0) return true; // e had been created by the current transaction and wasn't deleted 29 | 30 | if (tm.isCommitted(xmin)) { 31 | if (xmax == 0) return true; // e had been committed and wasn't deleted 32 | return xmax != xid && !tm.isCommitted(xmax); // e had been deleted by other transaction but wasn't committed 33 | } 34 | return false; 35 | } 36 | 37 | private static boolean repeatableRead(TransactionManager tm, Transaction t, Entry e) { 38 | long xid = t.xid; 39 | long xmin = e.getXmin(); 40 | long xmax = e.getXmax(); 41 | if (xmin == xid && xmax == 0) return true; // e had been created by the current transaction and wasn't deleted 42 | 43 | if (tm.isCommitted(xmin) && xmin < xid && !t.isInSnapShot(xmin)) { // e was committed before when the current transaction created 44 | if (xmax == 0) return true; // not deleted so far 45 | if (xmax != xid) { 46 | return !tm.isCommitted(xmax) || xmax > xid || t.isInSnapShot(xmax); // e was deleted by the transaction created after the current transaction 47 | } 48 | } 49 | return false; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/client/Client.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.client; 2 | 3 | import com.northeastern.edu.simpledb.transport.Package; 4 | import com.northeastern.edu.simpledb.transport.Packager; 5 | 6 | public class Client { 7 | private RoundTripper rt; 8 | 9 | public Client(Packager packager) { 10 | this.rt = new RoundTripper(packager); 11 | } 12 | 13 | public byte[] execute(byte[] stat) throws Exception { 14 | Package pkg = new Package(stat, null); 15 | Package resPkg = rt.roundTrip(pkg); 16 | if (resPkg.getErr() != null) { 17 | throw resPkg.getErr(); 18 | } 19 | return resPkg.getData(); 20 | } 21 | 22 | public void close() { 23 | try { 24 | rt.close(); 25 | } catch (Exception e) {} 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/client/Launcher.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.client; 2 | 3 | import com.northeastern.edu.simpledb.transport.Encoder; 4 | import com.northeastern.edu.simpledb.transport.Packager; 5 | import com.northeastern.edu.simpledb.transport.Transporter; 6 | 7 | import java.io.IOException; 8 | import java.net.InetAddress; 9 | import java.net.Socket; 10 | import java.net.UnknownHostException; 11 | 12 | public class Launcher { 13 | public static void main(String[] args) throws IOException { 14 | Socket socket = new Socket(InetAddress.getLocalHost().getHostAddress(), 9999); 15 | Encoder encoder = new Encoder(); 16 | Transporter transporter = new Transporter(socket); 17 | Packager packager = new Packager(transporter, encoder); 18 | 19 | Client client = new Client(packager); 20 | Shell shell = new Shell(client); 21 | shell.run(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/client/RoundTripper.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.client; 2 | 3 | import com.northeastern.edu.simpledb.transport.Package; 4 | import com.northeastern.edu.simpledb.transport.Packager; 5 | 6 | import java.io.IOException; 7 | 8 | public class RoundTripper { 9 | 10 | private Packager packager; 11 | 12 | public RoundTripper(Packager packager) { 13 | this.packager = packager; 14 | } 15 | 16 | public Package roundTrip(Package pkg) throws Exception { 17 | packager.send(pkg); 18 | return packager.receive(); 19 | } 20 | 21 | public void close() throws IOException { 22 | packager.close(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/client/Shell.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.client; 2 | 3 | import java.util.Scanner; 4 | 5 | public class Shell { 6 | 7 | private Client client; 8 | 9 | public Shell(Client client) { 10 | this.client = client; 11 | } 12 | 13 | public void run() { 14 | Scanner scanner = new Scanner(System.in); 15 | try { 16 | while (true) { 17 | System.out.print(":> "); 18 | String statStr = scanner.nextLine().strip(); 19 | if ("exit".equals(statStr) || "quit".equals(statStr)) break; 20 | try { 21 | byte[] res = client.execute(statStr.getBytes()); 22 | System.out.println(new String(res)); 23 | } catch (Exception e) { 24 | System.out.println(e.getMessage()); 25 | } 26 | } 27 | } finally { 28 | scanner.close(); 29 | client.close(); 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/common/Error.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.common; 2 | 3 | public class Error { 4 | 5 | // common 6 | public static final Exception FileNotExistsException = new RuntimeException("File does not exists!"); 7 | public static final Exception FileExistsException = new RuntimeException("File already exists!"); 8 | public static final Exception FileCannotRWException = new RuntimeException("File cannot read or write!"); 9 | 10 | public static final Exception CacheFullException = new RuntimeException("Cache is full!"); 11 | 12 | 13 | // dm 14 | public static final Exception MemTooSmallException = new RuntimeException("Memory too small!"); 15 | public static final Exception BadLogFileException = new RuntimeException("Bad log file!"); 16 | public static Exception DataTooLargeException = new RuntimeException("Data too large!"); 17 | public static Exception DataBaseBusyException = new RuntimeException("Database is too busy!"); 18 | 19 | // tm 20 | public static final Exception BadXIDFileException = new RuntimeException("Bad XID file!"); 21 | 22 | // vm 23 | public static Exception NullEntryException = new RuntimeException("Entry is null!"); 24 | public static Exception DeadLockException = new RuntimeException("Deadlock detected!"); 25 | 26 | // parser 27 | public static Exception InvalidCommandException = new RuntimeException("Command is invalid!"); 28 | public static Exception TableNoIndexException = new RuntimeException("Table has no index!"); 29 | 30 | // tbm 31 | public static Exception InvalidFieldException = new RuntimeException("Field is invalid!"); 32 | public static Exception DuplicatedTableException = new RuntimeException("Table already exists!"); 33 | public static Exception TableNotFoundException = new RuntimeException("Table does not exists!"); 34 | public static Exception FieldNotIndexedException = new RuntimeException("Field is not indexed!"); 35 | public static Exception InvalidLogOpException = new RuntimeException("Invalid logic operation!"); 36 | public static Exception FieldNotFoundException = new RuntimeException("Field does not exists!"); 37 | public static Exception ConcurrentUpdateException = new RuntimeException("Concurrent update happens!"); 38 | 39 | // c/s 40 | public static Exception InvalidPkgDataException = new RuntimeException("Package data is invalid!"); 41 | public static Exception NestedTransactionException = new RuntimeException("Nested transaction not supported!"); 42 | public static Exception NoTransactionException = new RuntimeException("Not in transaction!"); 43 | public static Exception InvalidMemException = new RuntimeException("Memory is invalid!"); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/transport/Encoder.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.transport; 2 | 3 | import com.google.common.primitives.Bytes; 4 | import com.northeastern.edu.simpledb.common.Error; 5 | 6 | import java.util.Arrays; 7 | 8 | public class Encoder { 9 | 10 | public byte[] encode(Package pkg) { 11 | if (pkg.getErr() != null) { 12 | Exception err = pkg.getErr(); 13 | String msg = "Internal Server Error!"; 14 | if (err.getMessage() != null) { 15 | msg = err.getMessage(); 16 | } 17 | return Bytes.concat(new byte[]{1}, msg.getBytes()); 18 | } else return Bytes.concat(new byte[]{0}, pkg.getData()); 19 | } 20 | 21 | public Package decode(byte[] data) throws Exception { 22 | if (data.length < 1) throw Error.InvalidPkgDataException; 23 | else if (data[0] == 0) return new Package(Arrays.copyOfRange(data, 1, data.length), null); 24 | else if (data[0] == 1) return new Package(null, new RuntimeException(new String(Arrays.copyOfRange(data, 1, data.length)))); 25 | else throw Error.InvalidPkgDataException; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/transport/Package.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.transport; 2 | 3 | public class Package { 4 | 5 | private byte[] data; 6 | private Exception err; 7 | 8 | public Package(byte[] data, Exception err) { 9 | this.data = data; 10 | this.err = err; 11 | } 12 | 13 | public byte[] getData() { 14 | return data; 15 | } 16 | 17 | public Exception getErr() { 18 | return err; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/transport/Packager.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.transport; 2 | 3 | import org.apache.commons.codec.DecoderException; 4 | 5 | import java.io.IOException; 6 | 7 | public class Packager { 8 | private Transporter transporter; 9 | private Encoder encoder; 10 | 11 | public Packager(Transporter transporter, Encoder encoder) { 12 | this.transporter = transporter; 13 | this.encoder = encoder; 14 | } 15 | 16 | public void send(Package pkg) throws IOException { 17 | byte[] data = encoder.encode(pkg); 18 | transporter.send(data); 19 | } 20 | 21 | public Package receive() throws Exception { 22 | byte[] data = transporter.receive(); 23 | return encoder.decode(data); 24 | } 25 | 26 | public void close() throws IOException { 27 | transporter.close(); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/northeastern/edu/simpledb/transport/Transporter.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.transport; 2 | 3 | import org.apache.commons.codec.DecoderException; 4 | import org.apache.commons.codec.binary.Hex; 5 | 6 | import java.io.*; 7 | import java.net.Socket; 8 | 9 | public class Transporter { 10 | 11 | private Socket socket; 12 | private BufferedReader reader; 13 | private BufferedWriter writer; 14 | 15 | public Transporter(Socket socket) throws IOException { 16 | this.socket = socket; 17 | this.reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); 18 | this.writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream())); 19 | } 20 | 21 | public byte[] receive() throws IOException, DecoderException { 22 | String line = reader.readLine(); 23 | if (line == null) close(); 24 | return hexDecode(line); 25 | } 26 | 27 | public void send(byte[] data) throws IOException { 28 | String raw = hexEncode(data); 29 | writer.write(raw); 30 | writer.flush(); 31 | } 32 | 33 | private String hexEncode(byte[] data) { 34 | return Hex.encodeHexString(data, true) + "\n"; 35 | } 36 | 37 | private byte[] hexDecode(String line) throws DecoderException { 38 | return Hex.decodeHex(line); 39 | } 40 | 41 | public void close() throws IOException { 42 | this.writer.close(); 43 | this.reader.close(); 44 | this.socket.close(); 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/common/AbstractCacheTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.common; 2 | 3 | import com.northeastern.edu.simpledb.common.Error; 4 | 5 | import java.util.Objects; 6 | import java.util.concurrent.*; 7 | 8 | import org.junit.jupiter.api.BeforeAll; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.io.*; 12 | 13 | import static org.junit.jupiter.api.Assertions.*; 14 | 15 | public class AbstractCacheTest { 16 | 17 | private static final String SERIALIZE_SUFFIX = ".ser"; 18 | 19 | private static final Integer THREAD_COUNT = 8; 20 | 21 | @BeforeAll 22 | static void prepareDataForTest() { 23 | 24 | for (int i = 1; i <= 2; i++) { 25 | // create an instance of the Data class 26 | Data data = new Data(); 27 | data.setContent(String.valueOf(i)); 28 | 29 | // specify the file path where you want to save the serialized object 30 | String filePath = data.getContent() + SERIALIZE_SUFFIX; 31 | 32 | // serialize the object and write it to the file 33 | try (FileOutputStream fileOut = new FileOutputStream(filePath); 34 | ObjectOutputStream objectOut = new ObjectOutputStream(fileOut)) { 35 | 36 | // write the object to the output stream 37 | objectOut.writeObject(data); 38 | 39 | System.out.println("Object has been serialized and written to " + filePath); 40 | 41 | } catch (IOException e) { 42 | e.printStackTrace(); 43 | } 44 | } 45 | } 46 | 47 | @Test 48 | void testCacheIsFull_throwCacheFullException() throws Exception { 49 | 50 | int maxResource = 1; 51 | MyAbstractCache cache = new MyAbstractCache(maxResource); 52 | 53 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT); 54 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT); 55 | 56 | Future[] futures = new Future[THREAD_COUNT]; 57 | 58 | assertThrows(Error.CacheFullException.getClass(), () -> { 59 | for (int i = 0; i < THREAD_COUNT; i++) { 60 | final int finalI = i; 61 | futures[i] = executorService.submit(() -> { 62 | countDownLatch.countDown(); 63 | try { 64 | countDownLatch.await(); 65 | Data data = cache.get(finalI % 2 == 0 ? 1L : 2L); 66 | if (data != null) System.out.println("data content = " + data.getContent()); 67 | } catch (Exception e) { 68 | throw new RuntimeException(e); 69 | } 70 | }); 71 | } 72 | 73 | for (int i = 0; i < THREAD_COUNT; i++) { 74 | try { 75 | futures[i].get(); 76 | } catch (InterruptedException | ExecutionException e) { 77 | throw new RuntimeException(e); 78 | } 79 | } 80 | }); 81 | 82 | executorService.shutdown(); 83 | executorService.awaitTermination(2, TimeUnit.SECONDS); 84 | } 85 | 86 | @Test 87 | void testMultiThreadGet_expectedNumberOfOutputEqualToThreadCount() throws Exception { 88 | int maxResource = 2; 89 | MyAbstractCache cache = new MyAbstractCache(maxResource); 90 | 91 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT); 92 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT); 93 | 94 | for (int i = 0; i < THREAD_COUNT; i++) { 95 | final int finalI = i; 96 | executorService.submit(() -> { 97 | countDownLatch.countDown(); 98 | try { 99 | countDownLatch.await(); 100 | Data data = cache.get(finalI % 2 == 0 ? 1L : 2L); 101 | if(data != null) System.out.println("data content = " + data.getContent()); 102 | } catch (Exception e) { 103 | assertEquals(Error.CacheFullException, e); 104 | } 105 | }); 106 | } 107 | 108 | executorService.shutdown(); 109 | executorService.awaitTermination(2, TimeUnit.SECONDS); 110 | } 111 | 112 | @Test 113 | void testCacheRelease_expectedOnlyOneSerialization() throws InterruptedException { 114 | int maxResource = 2; 115 | MyAbstractCache cache = new MyAbstractCache(maxResource); 116 | 117 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT); 118 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT); 119 | 120 | // submit multiple threads trying to access the same key concurrently 121 | for (int i = 0; i < THREAD_COUNT; i++) { 122 | final int finalI = i; 123 | executorService.submit(() -> { 124 | countDownLatch.countDown(); 125 | try { 126 | countDownLatch.await(); 127 | Data data = cache.get(1L); 128 | if (Thread.currentThread().getId() % 2 == 0) TimeUnit.SECONDS.sleep(2); 129 | System.out.println("Thread " + Thread.currentThread().getId() + " got data: " + data.getContent()); 130 | // update data 131 | data.setContent(String.valueOf(finalI)); 132 | // release the cache after processing 133 | cache.release(1L); 134 | } catch (Exception e) { 135 | throw new RuntimeException(e); 136 | } 137 | }); 138 | } 139 | 140 | executorService.shutdown(); 141 | executorService.awaitTermination(2, TimeUnit.SECONDS); 142 | } 143 | } 144 | 145 | class MyAbstractCache extends AbstractCache { 146 | 147 | private static final String SERIALIZE_SUFFIX = ".ser"; 148 | 149 | public MyAbstractCache(int maxResource) { 150 | super(maxResource); 151 | } 152 | 153 | @Override 154 | protected Data getForCache(long key) { 155 | Data data = null; 156 | 157 | // specify the file path where the serialized object is stored 158 | String filePath = key + SERIALIZE_SUFFIX; 159 | 160 | // deserialize the object from the file 161 | try (FileInputStream fileIn = new FileInputStream(filePath); 162 | ObjectInputStream objectIn = new ObjectInputStream(fileIn)) { 163 | 164 | // read the object from the input stream 165 | data = (Data) objectIn.readObject(); 166 | 167 | // access the content of the deserialized object 168 | System.out.println("Deserialized content: " + data.getContent()); 169 | 170 | } catch (IOException | ClassNotFoundException e) { 171 | e.printStackTrace(); 172 | } 173 | return data; 174 | } 175 | 176 | @Override 177 | protected void releaseForCache(Data obj) { 178 | 179 | // specify the file path where you want to save the serialized object 180 | String filePath = obj.getContent() + SERIALIZE_SUFFIX; 181 | 182 | // serialize the object and write it to the file 183 | try (FileOutputStream fileOut = new FileOutputStream(filePath); 184 | ObjectOutputStream objectOut = new ObjectOutputStream(fileOut)) { 185 | 186 | // write the object to the output stream 187 | objectOut.writeObject(obj); 188 | 189 | System.out.println("Object has been serialized and written to " + filePath); 190 | 191 | } catch (IOException e) { 192 | e.printStackTrace(); 193 | } 194 | } 195 | 196 | } 197 | 198 | class Data implements Serializable { 199 | 200 | public static final long SERIALIZE_ID = 123456789L; 201 | 202 | private String content; 203 | 204 | public String getContent() { 205 | return content; 206 | } 207 | 208 | public void setContent(String content) { 209 | this.content = content; 210 | } 211 | 212 | @Override 213 | public boolean equals(Object o) { 214 | if (this == o) return true; 215 | if (o == null || getClass() != o.getClass()) return false; 216 | Data data = (Data) o; 217 | return Objects.equals(content, data.content); 218 | } 219 | 220 | @Override 221 | public int hashCode() { 222 | return Objects.hash(content); 223 | } 224 | } -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/dm/DataManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.SubArray; 4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 7 | import com.northeastern.edu.simpledb.backend.utils.Panic; 8 | import com.northeastern.edu.simpledb.backend.utils.RandomUtil; 9 | import org.checkerframework.checker.units.qual.C; 10 | import org.junit.jupiter.api.AfterAll; 11 | import org.junit.jupiter.api.Assertions; 12 | import org.junit.jupiter.api.BeforeAll; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import java.io.File; 16 | import java.nio.charset.StandardCharsets; 17 | import java.security.SecureRandom; 18 | import java.util.*; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | import java.util.concurrent.CountDownLatch; 21 | import java.util.concurrent.locks.Lock; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX; 25 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX; 26 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX; 27 | 28 | public class DataManagerTest { 29 | 30 | private static final int workerNum = 10; 31 | private static final int tasksNum = 50; 32 | 33 | private static final int totalData = 10; 34 | static CountDownLatch cdl; 35 | static TransactionManager tm; 36 | 37 | static DataManger dm; 38 | 39 | static Lock lock; 40 | 41 | static List uids; 42 | 43 | static Random random = new SecureRandom(); 44 | static ConcurrentHashMap map; 45 | 46 | @BeforeAll 47 | static void setup() { 48 | uids = new ArrayList<>(); 49 | tm = TransactionManager.create("dm-test"); 50 | dm = DataMangerHandler.create("dm-test", PageCache.PAGE_SIZE * 10, tm); 51 | cdl = new CountDownLatch(workerNum); 52 | lock = new ReentrantLock(); 53 | map = new ConcurrentHashMap<>(); 54 | 55 | for (int i = 0; i < tasksNum; i++) { 56 | String s = UUID.randomUUID().toString(); 57 | map.put(i, s); 58 | } 59 | } 60 | 61 | @AfterAll 62 | static void cleanTestEnv() { 63 | new File("dm-test" + LOG_SUFFIX).delete(); 64 | new File("dm-test" + DB_SUFFIX).delete(); 65 | new File("dm-test" + XID_SUFFIX).delete(); 66 | } 67 | 68 | @Test 69 | void testMultiThread_expectedMapContainsAllElements() { 70 | for (int i = 0; i < workerNum; i++) { 71 | new Thread(() -> worker()).start(); 72 | } 73 | try { 74 | cdl.await(); 75 | } catch (InterruptedException e) { 76 | throw new RuntimeException(e); 77 | } 78 | } 79 | 80 | 81 | private static final int dataRotation = 50; 82 | 83 | private void worker() { 84 | try { 85 | for (int i = 0; i < tasksNum; i++) { 86 | int v; 87 | if ((v = random.nextInt() % 100) > dataRotation) { 88 | String s = map.get(random.nextInt(tasksNum)); 89 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8); 90 | long uid = dm.insert(0, bytes); 91 | try { 92 | lock.lock(); 93 | uids.add(uid); 94 | } finally { 95 | lock.unlock(); 96 | } 97 | } else { 98 | try { 99 | lock.lock(); 100 | if (uids.size() == 0) continue; 101 | } finally { 102 | lock.unlock(); 103 | } 104 | int tmp = Math.abs(random.nextInt()) % uids.size(); 105 | Long uid = uids.get(tmp); 106 | DataItem dataItem = dm.read(uid); 107 | 108 | dataItem.rLock(); 109 | SubArray sa = dataItem.data(); 110 | byte[] bytes = Arrays.copyOfRange(sa.raw, sa.start, sa.end); 111 | Assertions.assertTrue(map.contains(new String(bytes))); 112 | dataItem.rUnLock(); 113 | } 114 | } 115 | } catch (Exception e){ 116 | Panic.panic(e); 117 | } finally { 118 | cdl.countDown(); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/dm/RecoverTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm; 2 | 3 | import com.northeastern.edu.simpledb.backend.common.SubArray; 4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem; 6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger; 7 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 8 | import org.junit.jupiter.api.AfterAll; 9 | import org.junit.jupiter.api.Assertions; 10 | import org.junit.jupiter.api.BeforeAll; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.io.File; 14 | import java.lang.reflect.Field; 15 | import java.nio.charset.StandardCharsets; 16 | import java.util.Arrays; 17 | 18 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX; 19 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX; 20 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX; 21 | 22 | public class RecoverTest { 23 | 24 | private static DataManger dm; 25 | private static TransactionManager tm; 26 | 27 | private static long xid; 28 | 29 | @BeforeAll 30 | static void setup() { 31 | tm = TransactionManager.create("recover-test"); 32 | dm = DataMangerHandler.create("recover-test", PageCache.PAGE_SIZE * 10, tm); 33 | xid = tm.begin(); 34 | } 35 | 36 | @AfterAll 37 | static void cleanTestEnv() { 38 | new File("recover-test" + LOG_SUFFIX).delete(); 39 | new File("recover-test" + DB_SUFFIX).delete(); 40 | new File("recover-test" + XID_SUFFIX).delete(); 41 | } 42 | 43 | @Test 44 | void testRecover_expectedRedoWorksAndDataItemWithCorrespondingUidBecomesInvalid() { 45 | long uid; 46 | try { 47 | uid = dm.insert(xid, "hello db!".getBytes(StandardCharsets.UTF_8)); 48 | DataItem dataItem0 = dm.read(uid); 49 | dataItem0.rLock(); 50 | SubArray data0 = dataItem0.data(); 51 | dataItem0.rUnLock(); 52 | byte[] bytes = Arrays.copyOfRange(data0.raw, data0.start, data0.end); 53 | String actualStr = new String(bytes); 54 | Assertions.assertEquals("hello db!", actualStr); 55 | 56 | Field loggerFiled = dm.getClass().getDeclaredField("logger"); 57 | loggerFiled.setAccessible(true); 58 | Logger reloadedLog = Logger.open("recover-test"); 59 | loggerFiled.set(dm, reloadedLog); 60 | 61 | Field pageCacheField = dm.getClass().getDeclaredField("pageCache"); 62 | pageCacheField.setAccessible(true); 63 | PageCache pageCache = (PageCache) pageCacheField.get(dm); 64 | 65 | Recover.recover(tm, reloadedLog, pageCache); 66 | 67 | Assertions.assertNull(dm.read(uid)); 68 | 69 | } catch (Exception e) { 70 | throw new RuntimeException(e); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/dm/logger/LoggerTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.dm.logger; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger; 4 | import org.junit.jupiter.api.*; 5 | 6 | import java.io.IOException; 7 | import java.lang.reflect.Method; 8 | import java.nio.charset.StandardCharsets; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | 12 | @TestMethodOrder(MethodOrderer.OrderAnnotation.class) 13 | public class LoggerTest { 14 | 15 | private static final short OF_CHECKSUM = 4; 16 | private static Logger logger; 17 | 18 | private static final String LOG_NAME = "bin"; 19 | 20 | @BeforeAll 21 | static void setup() { 22 | logger = Logger.create(LOG_NAME); 23 | } 24 | 25 | @AfterAll 26 | static void cleanTestEnv() { 27 | try { 28 | Files.deleteIfExists(Path.of(LOG_NAME + Logger.LOG_SUFFIX)); 29 | } catch (IOException e) { 30 | throw new RuntimeException(e); 31 | } finally { 32 | logger.close(); 33 | } 34 | } 35 | 36 | @Test 37 | @Order(1) 38 | void testInit_expectedFCPositionEqualsToOffsetCheckSumAfterInit() { 39 | try { 40 | Method initMethod = logger.getClass().getDeclaredMethod("init"); 41 | initMethod.setAccessible(true); 42 | initMethod.invoke(logger); 43 | Assertions.assertEquals(OF_CHECKSUM, logger.getFc().position()); 44 | } catch (Exception e) { 45 | throw new RuntimeException(e); 46 | } 47 | } 48 | 49 | @Test 50 | @Order(2) 51 | void testLogOnce_expectedFCPositionEqualsToOffsetCheckSumAfterLog() { 52 | String sql = "select * from test"; 53 | byte[] bytes = sql.getBytes(StandardCharsets.UTF_8); 54 | logger.log(bytes); 55 | try { 56 | Assertions.assertEquals(OF_CHECKSUM, logger.getFc().position()); 57 | } catch (IOException e) { 58 | throw new RuntimeException(e); 59 | } 60 | } 61 | 62 | @Test 63 | @Order(3) 64 | void testLogOnce_expectedSameSQL() { 65 | String expected = "select * from test"; 66 | logger = Logger.open(LOG_NAME); 67 | byte[] log = logger.next(); 68 | assert log != null; 69 | String actual = new String(log); 70 | Assertions.assertEquals(expected, actual); 71 | } 72 | 73 | @Test 74 | @Order(4) 75 | void testLogMultipleTimes_expectedSameSQL() { 76 | String sql1 = "select * from test"; 77 | logger.log(sql1.getBytes(StandardCharsets.UTF_8)); 78 | 79 | String sql2 = "select * from test where a = 1"; 80 | logger.log(sql2.getBytes(StandardCharsets.UTF_8)); 81 | 82 | String sql3 = "update from test set b = 3 where a = 1"; 83 | logger.log(sql3.getBytes(StandardCharsets.UTF_8)); 84 | 85 | logger = Logger.open(LOG_NAME); 86 | 87 | logger.next(); 88 | 89 | byte[] log1 = logger.next(); 90 | assert log1 != null; 91 | String res1 = new String(log1); 92 | Assertions.assertEquals(sql1, res1); 93 | 94 | byte[] log2 = logger.next(); 95 | assert log2 != null; 96 | String res2 = new String(log2); 97 | Assertions.assertEquals(sql2, res2); 98 | 99 | byte[] log3 = logger.next(); 100 | assert log3 != null; 101 | String res3 = new String(log3); 102 | Assertions.assertEquals(sql3, res3); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/im/BPlusTreeTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.im; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler; 5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 7 | import org.junit.jupiter.api.AfterAll; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.io.File; 11 | import java.util.List; 12 | 13 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX; 14 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX; 15 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX; 16 | 17 | public class BPlusTreeTest { 18 | 19 | private static final String TEST_NAME = "bptree-test"; 20 | 21 | @AfterAll 22 | static void cleanTestEnv() { 23 | new File(TEST_NAME + LOG_SUFFIX).delete(); 24 | new File(TEST_NAME + DB_SUFFIX).delete(); 25 | new File(TEST_NAME + XID_SUFFIX).delete(); 26 | } 27 | 28 | @Test 29 | void testBuildTree_expectedNoException() throws Exception { 30 | DataManger dm = DataMangerHandler.create(TEST_NAME, PageCache.PAGE_SIZE * 10, TransactionManager.create(TEST_NAME)); 31 | 32 | long root = BPlusTree.create(dm); 33 | BPlusTree tree = BPlusTree.load(root, dm); 34 | 35 | int lim = 15; 36 | for (int i = 1; i <= lim; i++) { 37 | tree.insert(i, i); 38 | } 39 | for (int i = 1; i <= lim; i++) { 40 | List uids = tree.search(i); 41 | // System.out.println("============================="); 42 | assert uids.size() == 1; 43 | assert uids.get(0) == i; 44 | } 45 | 46 | /* 47 | tree.insert(16, 16); 48 | tree.insert(17, 17); 49 | tree.insert(18, 18); 50 | List uids = tree.search(6); 51 | */ 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/pageIndex/PageIndexTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.pageIndex; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 4 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageIndex; 5 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageInfo; 6 | import org.junit.jupiter.api.Test; 7 | 8 | public class PageIndexTest { 9 | 10 | @Test 11 | void testPageIndex_expectedNoException() { 12 | PageIndex pageIndex = new PageIndex(); 13 | int threshold = PageCache.PAGE_SIZE / 40; 14 | for (int i = 0; i < 40; i++) { 15 | pageIndex.add(i, i * threshold); 16 | pageIndex.add(i, i * threshold); 17 | pageIndex.add(i, i * threshold); 18 | } 19 | 20 | for (int i = 0; i < 39; i++) { 21 | PageInfo pageInfo = pageIndex.select(i * threshold); 22 | assert pageInfo != null; 23 | assert pageInfo.pageNumber == i + 1; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/parser/ParserTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser; 2 | 3 | import com.google.gson.Gson; 4 | import com.northeastern.edu.simpledb.backend.parser.statement.*; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import java.util.Arrays; 8 | 9 | class ParserTest { 10 | 11 | @Test 12 | void testCreate() throws Exception { 13 | String stat = "create table student id int32, name string, uid int64, (index name id uid)"; 14 | Object res = Parser.Parse(stat.getBytes()); 15 | Create create = (Create) res; 16 | assert "student".equals(create.tableName); 17 | System.out.println("Create"); 18 | for (int i = 0; i < create.fieldName.length; i++) { 19 | System.out.println(create.fieldName[i] + ":" + create.fieldType[i]); 20 | } 21 | System.out.println(Arrays.toString(create.index)); 22 | System.out.println("======================"); 23 | } 24 | 25 | @Test 26 | void testBegin() throws Exception { 27 | String stat = "begin isolation level read committed"; 28 | Object res = Parser.Parse(stat.getBytes()); 29 | Begin begin = (Begin) res; 30 | assert !begin.isRepeatableRead; 31 | 32 | stat = "begin"; 33 | res = Parser.Parse(stat.getBytes()); 34 | begin = (Begin) res; 35 | assert !begin.isRepeatableRead; 36 | 37 | stat = "begin isolation level repeatable read"; 38 | res = Parser.Parse(stat.getBytes()); 39 | begin = (Begin) res; 40 | assert begin.isRepeatableRead; 41 | } 42 | 43 | @Test 44 | void testRead() throws Exception { 45 | String stat = "select name, id, age from student where id > 1 and id < 4"; 46 | Object res = Parser.Parse(stat.getBytes()); 47 | Select select = (Select) res; 48 | assert "student".equals(select.tableName); 49 | Gson gson = new Gson(); 50 | System.out.println("Select"); 51 | System.out.println(gson.toJson(select.fields)); 52 | System.out.println(gson.toJson(select.where)); 53 | System.out.println("======================"); 54 | } 55 | 56 | @Test 57 | void testInsert() throws Exception { 58 | String stat = "insert into student values 5 \"jack\" 22"; 59 | Object res = Parser.Parse(stat.getBytes()); 60 | Insert insert = (Insert) res; 61 | Gson gson = new Gson(); 62 | System.out.println("Insert"); 63 | System.out.println(gson.toJson(insert)); 64 | System.out.println("======================"); 65 | } 66 | 67 | @Test 68 | void testDelete() throws Exception { 69 | String stat = "delete from student where name = \"jack\""; 70 | Object res = Parser.Parse(stat.getBytes()); 71 | Delete delete = (Delete) res; 72 | Gson gson = new Gson(); 73 | System.out.println("Delete"); 74 | System.out.println(gson.toJson(delete)); 75 | System.out.println("======================"); 76 | } 77 | 78 | @Test 79 | void testShow() throws Exception { 80 | String stat = "show"; 81 | Object res = Parser.Parse(stat.getBytes()); 82 | Show show = (Show) res; 83 | Gson gson = new Gson(); 84 | System.out.println("Show"); 85 | System.out.println(gson.toJson(show)); 86 | System.out.println("======================"); 87 | } 88 | 89 | @Test 90 | void testUpdate() throws Exception { 91 | String stat = "update student set name = \"jack\" where id = 5"; 92 | Object res = Parser.Parse(stat.getBytes()); 93 | Update update = (Update) res; 94 | Gson gson = new Gson(); 95 | System.out.println("Update"); 96 | System.out.println(gson.toJson(update)); 97 | System.out.println("======================"); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/parser/TokenizerTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.parser; 2 | 3 | import org.junit.jupiter.api.Assertions; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Test; 6 | import org.mockito.InjectMocks; 7 | import org.mockito.Mock; 8 | import org.mockito.MockitoAnnotations; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | import java.util.Objects; 12 | 13 | import static org.junit.jupiter.api.Assertions.*; 14 | import static org.mockito.Mockito.*; 15 | 16 | class TokenizerTest { 17 | 18 | @Mock 19 | Exception err; 20 | 21 | @InjectMocks 22 | Tokenizer tokenizer; 23 | 24 | @BeforeEach 25 | void setUp() { 26 | MockitoAnnotations.openMocks(this); 27 | } 28 | 29 | @Test 30 | void testValidInput_expectedNoException() { 31 | String s = "SELECT * FROM employee WHERE name = 'jack'"; 32 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8); 33 | Tokenizer tokenizer = new Tokenizer(bytes); 34 | 35 | assertDoesNotThrow(() -> { 36 | assertEquals("SELECT", tokenizer.peek()); 37 | tokenizer.pop(); 38 | 39 | assertEquals("*", tokenizer.peek()); 40 | tokenizer.pop(); 41 | }); 42 | } 43 | 44 | @Test 45 | void testInValidInput_expectedRuntimeException() { 46 | String s = " ^ "; 47 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8); 48 | Tokenizer tokenizer = new Tokenizer(bytes); 49 | 50 | assertThrows(RuntimeException.class, () -> { 51 | String token; 52 | while (!"".equals(token = tokenizer.peek())) { 53 | System.out.println("token = " + token); 54 | tokenizer.pop(); 55 | } 56 | }); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/tm/TransactionManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.tm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.page.Page; 4 | import org.checkerframework.checker.units.qual.C; 5 | import org.junit.jupiter.api.AfterAll; 6 | import org.junit.jupiter.api.BeforeAll; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import java.io.File; 10 | import java.nio.file.Files; 11 | import java.security.SecureRandom; 12 | import java.util.Map; 13 | import java.util.Random; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | import java.util.concurrent.CountDownLatch; 16 | import java.util.concurrent.locks.Lock; 17 | import java.util.concurrent.locks.ReentrantLock; 18 | 19 | public class TransactionManagerTest { 20 | 21 | static Random random = new SecureRandom(); 22 | 23 | private int transCnt = 0; 24 | private static final int noWorks = 300; 25 | private static final int noWorkers = 50; 26 | private static TransactionManager tm; 27 | private static Map transMap; 28 | private static Lock lock; 29 | private static CountDownLatch cdl; 30 | 31 | 32 | @BeforeAll 33 | static void setup() { 34 | tm = TransactionManager.create("tm-test"); 35 | lock = new ReentrantLock(); 36 | cdl = new CountDownLatch(noWorkers); 37 | transMap = new ConcurrentHashMap<>(); 38 | } 39 | 40 | @AfterAll 41 | static void cleanTestEnv() { 42 | assert new File("tm-test" + TransactionManager.XID_SUFFIX).delete(); 43 | } 44 | 45 | /** 46 | * let's suppose ConcurrentHashMap is thread-safe, randomly 47 | * starting a transaction, and put xid, status into ConcurrentHashMap 48 | * at same time, compare the state in ConcurrentHashMap with 49 | * state provided by Transaction Manager randomly. 50 | */ 51 | @Test 52 | void testMultiThread_expectedNoException() { 53 | for (int i = 0; i < noWorkers; i++) { 54 | Runnable r = () -> worker(); 55 | new Thread(r).run(); 56 | } 57 | try { 58 | cdl.await(); 59 | } catch (InterruptedException e) { 60 | e.printStackTrace(); 61 | } 62 | } 63 | 64 | private void worker() { 65 | boolean inTans = false; 66 | long transXID = 0; 67 | for (int i = 0; i < noWorks; i++) { 68 | int op = Math.abs(random.nextInt(6)); 69 | if (op == 0) { // do transaction 70 | lock.lock(); 71 | if (!inTans) { // starting a transaction synchronously in TM and ConcurrentHashMap 72 | long xid = tm.begin(); 73 | transMap.put(xid, (byte) 0); 74 | transCnt++; 75 | transXID = xid; 76 | inTans = true; 77 | } else { // determine if commit or abort transaction randomly 78 | int state = (random.nextInt(Integer.MAX_VALUE) % 2) + 1; 79 | switch (state) { 80 | case 1: 81 | tm.commit(transXID); 82 | break; 83 | case 2: 84 | tm.abort(transXID); 85 | break; 86 | } 87 | transMap.put(transXID, (byte) state); 88 | inTans = false; 89 | } 90 | lock.unlock(); 91 | } else { // check state 92 | lock.lock(); 93 | if (transCnt > 0) { 94 | long xid = (long) ((random.nextInt(Integer.MAX_VALUE) % transCnt) + 1); 95 | byte state = transMap.get(xid); 96 | boolean ok = false; 97 | switch (state) { 98 | case 0: 99 | ok = tm.isActive(xid); 100 | break; 101 | case 1: 102 | ok = tm.isCommitted(xid); 103 | break; 104 | case 2: 105 | ok = tm.isAborted(xid); 106 | break; 107 | } 108 | assert ok; 109 | } 110 | lock.unlock(); 111 | } 112 | } 113 | cdl.countDown(); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/vm/LockTableTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import org.junit.jupiter.api.Assertions; 5 | import org.junit.jupiter.api.BeforeEach; 6 | import org.junit.jupiter.api.Test; 7 | import org.mockito.InjectMocks; 8 | import org.mockito.Mock; 9 | import org.mockito.MockitoAnnotations; 10 | import org.mockito.Spy; 11 | 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.concurrent.locks.Lock; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertThrows; 19 | 20 | class LockTableTest { 21 | @Spy 22 | Map> x2u = new HashMap<>(); 23 | @InjectMocks 24 | LockTable lockTable; 25 | 26 | @BeforeEach 27 | void setUp() { 28 | MockitoAnnotations.openMocks(this); 29 | } 30 | 31 | /* 32 | eg: 33 | dfs(xid2) 34 | xid2 try to acquire uid1, but uid1 is holding by xid1 35 | dfs(xid1) 36 | xid1 is waiting for uid2, but uid2 is holding by xid2 37 | */ 38 | @Test 39 | void testSingleAdd_expectedRuntimeException() throws Exception { 40 | try { 41 | lockTable.add(1, 1); 42 | lockTable.add(2, 2); 43 | lockTable.add(1, 2); 44 | } catch (Exception e) { 45 | Panic.panic(e); 46 | } 47 | 48 | assertThrows(RuntimeException.class, () -> lockTable.add(2, 1)); 49 | } 50 | 51 | @Test 52 | void testRemove_expectedNoException() { 53 | try { 54 | lockTable.add(1, 1); 55 | lockTable.add(1, 2); 56 | lockTable.add(1, 3); 57 | lockTable.add(1, 4); 58 | lockTable.add(2, 4); 59 | lockTable.remove(1); 60 | } catch (Exception e) { 61 | Panic.panic(e); 62 | } 63 | } 64 | 65 | @Test 66 | void testMultiAdd_expectedRuntimeException() { 67 | for (long i = 1; i <= 100; i ++) { 68 | try { 69 | Lock o = lockTable.add(i, i); 70 | if(o != null) { 71 | Runnable r = () -> { 72 | o.lock(); 73 | o.unlock(); 74 | }; 75 | new Thread(r).start(); 76 | } 77 | } catch (Exception e) { 78 | Panic.panic(e); 79 | } 80 | } 81 | 82 | for (long i = 1; i <= 99; i ++) { 83 | try { 84 | Lock o = lockTable.add(i, i+1); 85 | if(o != null) { 86 | Runnable r = () -> { 87 | o.lock(); 88 | o.unlock(); 89 | }; 90 | new Thread(r).start(); 91 | } 92 | } catch (Exception e) { 93 | Panic.panic(e); 94 | } 95 | } 96 | 97 | assertThrows(RuntimeException.class, () -> lockTable.add(100, 1)); 98 | lockTable.remove(23); 99 | 100 | try { 101 | lockTable.add(100, 1); 102 | } catch (Exception e) { 103 | Panic.panic(e); 104 | } 105 | } 106 | 107 | 108 | @Test 109 | void testPutIntoList_expectedNoException() { 110 | x2u.computeIfAbsent(1L, k -> new ArrayList<>()).add(0, 1L); 111 | assert x2u.size() > 0 && x2u.get(1L) != null && x2u.get(1L).size() == 1; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/backend/vm/VersionManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.backend.vm; 2 | 3 | import com.northeastern.edu.simpledb.backend.dm.DataManger; 4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler; 5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache; 6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager; 7 | import org.junit.jupiter.api.*; 8 | import org.mockito.InjectMocks; 9 | import org.mockito.MockitoAnnotations; 10 | 11 | import java.io.File; 12 | import java.nio.charset.StandardCharsets; 13 | 14 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX; 15 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX; 16 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX; 17 | 18 | class VersionManagerTest { 19 | 20 | private static final String TEST_NAME = "vm-test"; 21 | 22 | static TransactionManager tm; 23 | static DataManger dm; 24 | static VersionManagerHandler vm; 25 | 26 | @BeforeEach 27 | void setUp() { 28 | MockitoAnnotations.openMocks(this); 29 | } 30 | 31 | @BeforeAll 32 | static void setUpEnv() { 33 | tm = TransactionManager.create(TEST_NAME); 34 | dm = DataMangerHandler.create(TEST_NAME, PageCache.PAGE_SIZE * 10, tm); 35 | vm = VersionManagerHandler.newVersionManager(tm, dm); 36 | } 37 | 38 | @AfterAll 39 | static void cleanTestEnv() { 40 | new File(TEST_NAME + LOG_SUFFIX).delete(); 41 | new File(TEST_NAME + DB_SUFFIX).delete(); 42 | new File(TEST_NAME + XID_SUFFIX).delete(); 43 | } 44 | 45 | @Test 46 | void testRead_expectedNoException() throws Exception { 47 | long xid = vm.begin(1); 48 | long uid = vm.insert(xid, "version manager read test".getBytes(StandardCharsets.UTF_8)); 49 | Assertions.assertEquals(new String(vm.read(xid, uid)), "version manager read test"); 50 | vm.commit(xid); 51 | } 52 | 53 | @Test 54 | void testDelete_expectedNoException() throws Exception { 55 | long xid = vm.begin(1); 56 | long uid = vm.insert(xid, "version manager delete test".getBytes(StandardCharsets.UTF_8)); 57 | Assertions.assertEquals(vm.delete(xid, uid), Boolean.TRUE); 58 | Assertions.assertNull(vm.read(xid, uid)); 59 | vm.commit(xid); 60 | } 61 | 62 | @Test 63 | void testAbort_expectedRollBackSuccess() throws Exception { 64 | long xid = vm.begin(1); 65 | long uid = vm.insert(xid, "version manager abort test".getBytes(StandardCharsets.UTF_8)); 66 | vm.abort(xid); 67 | Assertions.assertNull(vm.read(xid, uid)); 68 | } 69 | 70 | } -------------------------------------------------------------------------------- /src/test/java/com/northeastern/edu/simpledb/transport/PackageTest.java: -------------------------------------------------------------------------------- 1 | package com.northeastern.edu.simpledb.transport; 2 | 3 | import com.northeastern.edu.simpledb.backend.utils.Panic; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.net.InetAddress; 7 | import java.net.ServerSocket; 8 | import java.net.Socket; 9 | 10 | public class PackageTest { 11 | 12 | @Test 13 | void testPackager() throws Exception { 14 | new Thread(() -> { 15 | try { 16 | ServerSocket welcomeSocket = new ServerSocket(10888); 17 | Socket socket = welcomeSocket.accept(); 18 | Transporter t = new Transporter(socket); 19 | Encoder e = new Encoder(); 20 | Packager p = new Packager(t, e); 21 | 22 | Package one = p.receive(); 23 | assert "pkg1 test".equals(new String(one.getData())); 24 | Package two = p.receive(); 25 | assert "pkg2 test".equals(new String(two.getData())); 26 | 27 | p.send(new Package("pkg3 test".getBytes(), null)); 28 | welcomeSocket.close(); 29 | } catch (Exception e) { 30 | Panic.panic(e); 31 | } 32 | }).start(); 33 | 34 | Socket socket = new Socket(InetAddress.getLocalHost().getHostAddress(), 10888); 35 | Transporter t = new Transporter(socket); 36 | Encoder e = new Encoder(); 37 | Packager p = new Packager(t, e); 38 | 39 | p.send(new Package("pkg1 test".getBytes(), null)); 40 | p.send(new Package("pkg2 test".getBytes(), null)); 41 | 42 | Package three = p.receive(); 43 | assert "pkg3 test".equals(new String(three.getData())); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /start-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | PACKAGE_NAME=com.northeastern.edu.simpledb.client.Launcher 6 | 7 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME" -------------------------------------------------------------------------------- /start-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | PACKAGE_NAME=com.northeastern.edu.simpledb.backend.Launcher 6 | 7 | mvn clean compile 8 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME" -Dexec.args="-create simple-db" 9 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME" -Dexec.args="-open simple-db" --------------------------------------------------------------------------------