├── .gitignore
├── README.md
├── pom.xml
├── src
├── main
│ └── java
│ │ └── com
│ │ └── northeastern
│ │ └── edu
│ │ └── simpledb
│ │ ├── backend
│ │ ├── Launcher.java
│ │ ├── common
│ │ │ ├── AbstractCache.java
│ │ │ └── SubArray.java
│ │ ├── dm
│ │ │ ├── DataManger.java
│ │ │ ├── DataMangerHandler.java
│ │ │ ├── Recover.java
│ │ │ ├── cache
│ │ │ │ ├── PageCache.java
│ │ │ │ └── PageCacheHandler.java
│ │ │ ├── dataItem
│ │ │ │ ├── AbstractDataItem.java
│ │ │ │ └── DataItem.java
│ │ │ ├── logger
│ │ │ │ ├── AbstractLogger.java
│ │ │ │ └── Logger.java
│ │ │ ├── page
│ │ │ │ ├── AbstractPage.java
│ │ │ │ ├── L1Page.java
│ │ │ │ ├── Page.java
│ │ │ │ └── SecondaryPage.java
│ │ │ └── pageIndex
│ │ │ │ ├── PageIndex.java
│ │ │ │ └── PageInfo.java
│ │ ├── im
│ │ │ ├── BPlusTree.java
│ │ │ └── Node.java
│ │ ├── parser
│ │ │ ├── Parser.java
│ │ │ ├── Tokenizer.java
│ │ │ └── statement
│ │ │ │ ├── Abort.java
│ │ │ │ ├── Begin.java
│ │ │ │ ├── Commit.java
│ │ │ │ ├── Create.java
│ │ │ │ ├── Delete.java
│ │ │ │ ├── Drop.java
│ │ │ │ ├── Insert.java
│ │ │ │ ├── Select.java
│ │ │ │ ├── Show.java
│ │ │ │ ├── SingleExpression.java
│ │ │ │ ├── Update.java
│ │ │ │ └── Where.java
│ │ ├── server
│ │ │ ├── Executor.java
│ │ │ └── Server.java
│ │ ├── tbm
│ │ │ ├── AbstractTableManager.java
│ │ │ ├── BeginRes.java
│ │ │ ├── Booter.java
│ │ │ ├── Field.java
│ │ │ ├── FieldCalRes.java
│ │ │ ├── Table.java
│ │ │ └── TableManager.java
│ │ ├── tm
│ │ │ ├── AbstractTransactionManager.java
│ │ │ └── TransactionManager.java
│ │ ├── utils
│ │ │ ├── Panic.java
│ │ │ ├── ParseStringRes.java
│ │ │ ├── Parser.java
│ │ │ ├── RandomUtil.java
│ │ │ └── Types.java
│ │ └── vm
│ │ │ ├── Entry.java
│ │ │ ├── LockTable.java
│ │ │ ├── Transaction.java
│ │ │ ├── VersionManager.java
│ │ │ ├── VersionManagerHandler.java
│ │ │ └── Visibility.java
│ │ ├── client
│ │ ├── Client.java
│ │ ├── Launcher.java
│ │ ├── RoundTripper.java
│ │ └── Shell.java
│ │ ├── common
│ │ └── Error.java
│ │ └── transport
│ │ ├── Encoder.java
│ │ ├── Package.java
│ │ ├── Packager.java
│ │ └── Transporter.java
└── test
│ └── java
│ └── com
│ └── northeastern
│ └── edu
│ └── simpledb
│ ├── backend
│ ├── common
│ │ └── AbstractCacheTest.java
│ ├── dm
│ │ ├── DataManagerTest.java
│ │ ├── RecoverTest.java
│ │ └── logger
│ │ │ └── LoggerTest.java
│ ├── im
│ │ └── BPlusTreeTest.java
│ ├── pageIndex
│ │ └── PageIndexTest.java
│ ├── parser
│ │ ├── ParserTest.java
│ │ └── TokenizerTest.java
│ ├── tm
│ │ └── TransactionManagerTest.java
│ └── vm
│ │ ├── LockTableTest.java
│ │ └── VersionManagerTest.java
│ └── transport
│ └── PackageTest.java
├── start-client.sh
└── start-server.sh
/.gitignore:
--------------------------------------------------------------------------------
1 | # IDEA file
2 | .idea/*
3 |
4 | # Temporary serialized file
5 | *.ser
6 |
7 | # DB data
8 | *.bt
9 | *.db
10 | *.log
11 | *.xid
12 |
13 | # Compiled file
14 | target/*
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Simple DB
2 | ______
3 | - Data reliability and data recovery
4 | - Two-stage locking protocol (2PL) implements serializable scheduling and MVCC
5 | - Two transaction isolation levels (read committed and repeatable read)
6 | - Deadlock handling
7 | - Simple table, field management and SQL parsing
8 | - Socket-based server and client
9 | ## How to start
10 | ______
11 | ### Pre-requirement
12 | - OpenJDK 11
13 | - Apache Maven 3.9.5
14 | ### Shell
15 | - step1
16 | ```bash
17 | chmod +x ./start-server.sh ./start-client.sh
18 | ```
19 | - step2
20 | ```bash
21 | ./start-server.sh
22 | ./start-client.sh
23 | ```
24 | ## Dependency graph
25 | ______
26 | ```mermaid
27 | graph TD;
28 | IM -->|Dependency| DM;
29 | DM -->|Dependency| TM;
30 | TBM -->|Dependency| VM;
31 | TBM -->|Dependency| IM;
32 | VM -->|Dependency| TM;
33 | VM -->|Dependency| DM;
34 | ```
35 | ## Transaction Manager
36 | ______
37 | ### Transaction Manager
38 | - builder pattern
39 | - responsible for state of transaction
40 | - maintaining a file starting with a `xid`
41 | ## Data Manager
42 | ______
43 | ### Abstract Cache
44 | - template pattern
45 | - reference counting
46 | ### Logger
47 | - iterator pattern
48 | - format
49 | - log file: `[Xchecksum][log1][log2][log3]...[logN][BadTail]`
50 | - log: `[Size][Checksum][Data]`
51 | - data
52 | - update: `[DataType: 1byte][Xid: 8bytes][Uid: 8bytes][OldRaw][LatestRaw]`
53 | - uid: `[PageNumber: 4bytes][Offset: 4bytes]`
54 | - insert: `[DataType: 1byte][Xid: 8bytes][PageNumber: 4bytes][Offset: 2bytes][Raw]`
55 | - process
56 | - redo: scan log and process redo logs which state is committed
57 | - undo: scan log and process undo logs which state is active reversely
58 | - logging
59 | 1. wrap data to log
60 | 2. write log to log file
61 | 3. update check sum
62 | 4. flush to disk
63 | ### Page
64 | - format
65 | - level1 page
66 | - when db start up, write random bytes from `OF_VC` to `OF_VC + LEN_VC - 1` byte
67 | - when db close, copy random bytes from `OF_VC + LEN_VC` to `OF_VC + 2 * LEN_VC - 1` byte
68 | - used to determine whether the database was shut down normally in the last time
69 | - secondary page
70 | - `[Offset: 2bytes][Raw]`
71 | - stored FSO (free space offset)
72 | - used to process raw data as secondary page when page cache need it in the process of initialization of DataManager
73 | ### Page Index
74 | - splitting the secondary page into four-ty slots
75 | > the page index caches the free space of each page, it is used to quickly find a page with suitable space when the upper module performs an insertion operation without checking the information of each page from the disk or cache.
76 | ### Data Item
77 | - format
78 | - data item: `[ValidFlag: 1byte][DataSize: 3bytes][Raw]`
79 | ### Data Manager
80 | - feature: providing data accessing service for the upper layer, eg: vm, im
81 | - method
82 | - insert()
83 | - read()
84 | - update()
85 | ## Index Manager
86 | ______
87 | ### Index Manager
88 | - format
89 | - Node: `[NumberOfKeys][SiblingUid][Son0][Key0][Son1][Key1]...[SonN][KeyN]`
90 | - insert()
91 | 1. assume `BALANCE` is `3`
92 | ```mermaid
93 | graph TD
94 | A(root) --> B(4)
95 | A --> C(7)
96 | A --> D(10)
97 | A --> E(13)
98 | B --> F(1)
99 | B --> G(2)
100 | B --> H(3)
101 | C --> I(4)
102 | C --> J(5)
103 | C --> K(6)
104 | D --> L(7)
105 | D --> M(8)
106 | D --> N(9)
107 | E --> O(10)
108 | E --> P(11)
109 | E --> Q(12)
110 | A --> U(MAX)
111 | U --> R(13)
112 | U --> S(14)
113 | U --> T(15)
114 | U --> V(16)
115 | U --> W(17)
116 | ```
117 | 2. after insert `18`, `MAX` will be split, `16` will be created for `13, 14, 15`, then `MAX` will be created for `16, 17, 18`
118 | ```mermaid
119 | graph TD
120 | A(root) --> B(4)
121 | A --> C(7)
122 | A --> D(10)
123 | A --> E(13)
124 | B --> F(1)
125 | B --> G(2)
126 | B --> H(3)
127 | C --> I(4)
128 | C --> J(5)
129 | C --> K(6)
130 | D --> L(7)
131 | D --> M(8)
132 | D --> N(9)
133 | E --> O(10)
134 | E --> P(11)
135 | E --> Q(12)
136 | A --> U(MAX)
137 | U --> R(13)
138 | U --> S(14)
139 | U --> T(15)
140 | U --> V(16)
141 | U --> W(17)
142 | U --> X(18)
143 | ```
144 | ```mermaid
145 | graph TD
146 | A(root) --> B(4)
147 | A --> C(7)
148 | A --> D(10)
149 | A --> E(13)
150 | B --> F(1)
151 | B --> G(2)
152 | B --> H(3)
153 | C --> I(4)
154 | C --> J(5)
155 | C --> K(6)
156 | D --> L(7)
157 | D --> M(8)
158 | D --> N(9)
159 | E --> O(10)
160 | E --> P(11)
161 | E --> Q(12)
162 | A --> Y(16)
163 | Y --> R(13)
164 | Y --> S(14)
165 | Y --> T(15)
166 | U(MAX) --> V(16)
167 | U --> W(17)
168 | U --> X(18)
169 | ```
170 | 3. `MAX` will be inserted into `root`, so the number of son of `root` exceed limitation
171 | 4. `root` will be split, then another one `MAX` created
172 | ```mermaid
173 | graph TD
174 | A(root) --> B(4)
175 | A --> C(7)
176 | A --> D(10)
177 | B --> F(1)
178 | B --> G(2)
179 | B --> H(3)
180 | C --> I(4)
181 | C --> J(5)
182 | C --> K(6)
183 | D --> L(7)
184 | D --> M(8)
185 | D --> N(9)
186 | E --> O(10)
187 | E --> P(11)
188 | E --> Q(12)
189 | Y --> R(13)
190 | Y --> S(14)
191 | Y --> T(15)
192 | U(MAX) --> V(16)
193 | U --> W(17)
194 | U --> X(18)
195 | Z(MAX) --> E(13)
196 | Z --> Y(16)
197 | Z --> U
198 | ```
199 | 5. create a `newRoot` and replace old root with it
200 | ```mermaid
201 | graph TD
202 | A(13) --> B(4)
203 | A --> C(7)
204 | A --> D(10)
205 | B --> F(1)
206 | B --> G(2)
207 | B --> H(3)
208 | C --> I(4)
209 | C --> J(5)
210 | C --> K(6)
211 | D --> L(7)
212 | D --> M(8)
213 | D --> N(9)
214 | E --> O(10)
215 | E --> P(11)
216 | E --> Q(12)
217 | U --> R(13)
218 | U --> S(14)
219 | U --> T(15)
220 | W --> X(16)
221 | W --> Y(17)
222 | W --> Z(18)
223 | V(MAX) --> E(13)
224 | V --> U(16)
225 | V --> W(MAX)
226 | newRoot --> A
227 | newRoot --> V
228 | ```
229 | - search()
230 | ## Version Manager
231 | ______
232 | ### Version Manager
233 | - two-phase locking & multi-version concurrency control
234 | - deadlock detection by depth first search
235 | - format
236 | - entry: `[Xmin][Xmax][Data]`
237 | ## Table Manager
238 | ______
239 | ### Parser
240 | - lexical parser and grammar parser
241 | - finite state machine
242 | ```mermaid
243 | graph TD
244 | INIT -->|Symbol| IN_SYMBOL
245 | INIT -->|QuoteChar| IN_QUOTE
246 | INIT -->|Alphabet/Digit| IN_TOKEN
247 | IN_SYMBOL -->|Any| INIT
248 | IN_QUOTE -->|Any| IN_QUOTE
249 | IN_QUOTE -->|QuoteChar| INIT
250 | IN_TOKEN -->|Alphabet/Digit| IN_TOKEN
251 | IN_TOKEN -->|Blank| INIT
252 | ```
253 | ### Table Manager
254 | - format
255 | - field: `[FieldName][TypeName][IndexUid]`
256 | - `[StringLength][StringData]`
257 | - table: `[TableName][NextTable][Fiedl1Uid][Field2Uid]...[FieldNUid]`
258 | - safe/atomic update of files via temporary files
259 | > ref: https://mozillazg.com/2018/05/a-way-to-atomic-write-or-safe-update-a-file.html
260 | - when creating a new table, the header interpolation method is used, so the Booter file needs to be updated every time a table is created.
261 | ## Transport
262 | ______
263 | ### C/S
264 | - A special binary format is used for communication between the client and the server.
265 | - In order to avoid problems caused by special characters, the data will be converted into a hexadecimal string (Hex String).
266 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 | 4.0.0
6 |
7 | com.northeastern.edu
8 | simple-db
9 | 1.0-SNAPSHOT
10 | jar
11 |
12 | simple-db Maven Webapp
13 |
14 |
15 | UTF-8
16 | 11
17 | 11
18 |
19 |
20 |
21 |
22 | com.google.guava
23 | guava
24 | 32.1.2-jre
25 |
26 |
27 | commons-codec
28 | commons-codec
29 | 1.14
30 |
31 |
32 | commons-cli
33 | commons-cli
34 | 1.5.0
35 |
36 |
37 | org.junit.jupiter
38 | junit-jupiter-api
39 | 5.9.1
40 | test
41 |
42 |
43 | org.mockito
44 | mockito-core
45 | 4.0.0
46 | test
47 |
48 |
49 | com.google.code.gson
50 | gson
51 | 2.8.9
52 | test
53 |
54 |
55 |
56 |
57 | simple-db
58 |
59 |
60 |
61 | maven-clean-plugin
62 | 3.1.0
63 |
64 |
65 |
66 | maven-resources-plugin
67 | 3.0.2
68 |
69 |
70 | maven-compiler-plugin
71 | 3.8.0
72 |
73 |
74 | maven-surefire-plugin
75 | 2.22.1
76 |
77 |
78 | maven-war-plugin
79 | 3.2.2
80 |
81 |
82 | maven-install-plugin
83 | 2.5.2
84 |
85 |
86 | maven-deploy-plugin
87 | 2.8.2
88 |
89 |
90 |
91 |
92 |
93 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/Launcher.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler;
5 | import com.northeastern.edu.simpledb.backend.server.Server;
6 | import com.northeastern.edu.simpledb.backend.tbm.TableManager;
7 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
8 | import com.northeastern.edu.simpledb.backend.utils.Panic;
9 | import com.northeastern.edu.simpledb.backend.vm.VersionManager;
10 | import com.northeastern.edu.simpledb.common.Error;
11 | import org.apache.commons.cli.CommandLine;
12 | import org.apache.commons.cli.DefaultParser;
13 | import org.apache.commons.cli.Options;
14 | import org.apache.commons.cli.ParseException;
15 |
16 | public class Launcher {
17 |
18 | public static final int port = 9999;
19 |
20 | public static final long DEFALUT_MEM = (1<<20)*64;
21 |
22 | public static final long KB = 1 << 10;
23 | public static final long MB = 1 << 20;
24 | public static final long GB = 1 << 30;
25 |
26 | public static void main(String[] args) throws ParseException {
27 | Options options = new Options();
28 | options.addOption("open", true, "-open DBPath");
29 | options.addOption("create", true, "-create DBPath");
30 | options.addOption("mem", true, "-mem 64MB");
31 | DefaultParser parser = new DefaultParser();
32 | CommandLine cmd = parser.parse(options, args);
33 |
34 | if(cmd.hasOption("open")) {
35 | openDB(cmd.getOptionValue("open"), parseMem(cmd.getOptionValue("mem")));
36 | return;
37 | }
38 | if(cmd.hasOption("create")) {
39 | createDB(cmd.getOptionValue("create"));
40 | return;
41 | }
42 | System.out.println("Usage: launcher (open|create) DBPath");
43 | }
44 |
45 | private static void createDB(String path) {
46 | TransactionManager tm = TransactionManager.create(path);
47 | DataManger dm = DataMangerHandler.create(path, DEFALUT_MEM, tm);
48 | VersionManager vm = new VersionManager(tm, dm);
49 | TableManager.create(path, vm, dm);
50 | tm.close();
51 | dm.close();
52 | }
53 |
54 | private static void openDB(String path, long mem) {
55 | TransactionManager tm = TransactionManager.open(path);
56 | DataManger dm = DataMangerHandler.open(path, mem, tm);
57 | VersionManager vm = new VersionManager(tm, dm);
58 | TableManager tbm = TableManager.open(path, vm, dm);
59 | new Server(port, tbm).start();
60 | }
61 |
62 | private static long parseMem(String memStr) {
63 | if(memStr == null || "".equals(memStr)) {
64 | return DEFALUT_MEM;
65 | }
66 | if(memStr.length() < 2) {
67 | Panic.panic(Error.InvalidMemException);
68 | }
69 | String unit = memStr.substring(memStr.length()-2);
70 | long memNum = Long.parseLong(memStr.substring(0, memStr.length()-2));
71 | switch(unit) {
72 | case "KB":
73 | return memNum*KB;
74 | case "MB":
75 | return memNum*MB;
76 | case "GB":
77 | return memNum*GB;
78 | default:
79 | Panic.panic(Error.InvalidMemException);
80 | }
81 | return DEFALUT_MEM;
82 | }
83 |
84 |
85 | }
86 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/common/AbstractCache.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.common;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 | import java.util.Set;
6 | import java.util.concurrent.ConcurrentHashMap;
7 | import java.util.concurrent.locks.Condition;
8 | import java.util.concurrent.locks.Lock;
9 | import java.util.concurrent.locks.ReentrantLock;
10 |
11 | import com.northeastern.edu.simpledb.common.Error;
12 |
13 | /**
14 | * AbstractCache is a reference counting strategy for caching
15 | */
16 | public abstract class AbstractCache {
17 |
18 | // the actual cache for data
19 | private Map cache;
20 |
21 | // a counter for the reference
22 | private Map references;
23 |
24 | // whether the resource is being obtained by other thread
25 | private Map getting;
26 |
27 | // the maximum number of data in cache
28 | private int maxResource;
29 |
30 | // the number of elements in the cache
31 | private int count = 0;
32 |
33 | private Lock lock;
34 |
35 | private Condition condition;
36 |
37 | public AbstractCache(int maxResource) {
38 | this.maxResource = maxResource;
39 | this.cache = new ConcurrentHashMap<>();
40 | this.references = new ConcurrentHashMap<>();
41 | this.getting = new ConcurrentHashMap<>();
42 | this.lock = new ReentrantLock();
43 | this.condition = lock.newCondition();
44 | }
45 |
46 | // obtain data based on key
47 | protected T get(long key) throws Exception {
48 |
49 | lock.lock();
50 |
51 | try {
52 | // the thread starts waiting while the data the thread wants is being loaded into the cache
53 | while (getting.containsKey(key)) {
54 | try {
55 | condition.await();
56 | } catch (InterruptedException e) {
57 | throw new RuntimeException(e);
58 | }
59 | }
60 |
61 | if (cache.containsKey(key)) {
62 | T obj = cache.get(key);
63 | references.put(key, references.get(key) + 1);
64 | return obj;
65 | }
66 |
67 | if(maxResource > 0 && count == maxResource) {
68 | throw Error.CacheFullException;
69 | }
70 |
71 | count++;
72 | getting.put(key, true);
73 | } catch (Exception e) {
74 | throw e;
75 | } finally {
76 | lock.unlock();
77 | }
78 |
79 |
80 | T obj = null;
81 | try {
82 | obj = getForCache(key);
83 | } catch(Exception e) {
84 | lock.lock();
85 | try {
86 | count--;
87 | getting.remove(key);
88 | } finally {
89 | lock.unlock();
90 | }
91 | throw e;
92 | }
93 |
94 | lock.lock();
95 | try {
96 | getting.remove(key);
97 | cache.put(key, obj);
98 | references.put(key, 1);
99 | // after data is loaded into the cache, notify all threads is sleeping before
100 | condition.signalAll();
101 | } finally {
102 | lock.unlock();
103 | }
104 |
105 | return obj;
106 | }
107 |
108 | // forcefully release a cache based on key
109 | protected void release(long key) {
110 | lock.lock();
111 | try {
112 | int ref = references.get(key) - 1;
113 | if (ref < 0) return ;
114 | if(ref == 0) {
115 | T obj = cache.get(key);
116 | releaseForCache(obj);
117 | references.remove(key);
118 | cache.remove(key);
119 | count--;
120 | } else {
121 | references.put(key, ref);
122 | }
123 | } finally {
124 | lock.unlock();
125 | }
126 | }
127 |
128 | // turn off caching and write back all resources
129 | protected void close() {
130 | lock.lock();
131 | try {
132 | Set keys = cache.keySet();
133 | for (long key : keys) {
134 | T obj = cache.get(key);
135 | releaseForCache(obj);
136 | references.remove(key);
137 | cache.remove(key);
138 | }
139 | } finally {
140 | lock.unlock();
141 | }
142 | }
143 |
144 | // default behavior to perform if cache misses
145 | protected abstract T getForCache(long key) throws Exception;
146 |
147 | // default write back behavior when cache eviction
148 | protected abstract void releaseForCache(T obj);
149 | }
150 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/common/SubArray.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.common;
2 |
3 | public class SubArray {
4 | public byte[] raw;
5 | public int start;
6 | public int end;
7 |
8 | public SubArray(byte[] raw, int start, int end) {
9 | this.raw = raw;
10 | this.start = start;
11 | this.end = end;
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/DataManger.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache;
4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger;
7 | import com.northeastern.edu.simpledb.backend.dm.page.L1Page;
8 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
9 | import com.northeastern.edu.simpledb.backend.dm.page.SecondaryPage;
10 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageIndex;
11 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageInfo;
12 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
13 | import com.northeastern.edu.simpledb.backend.utils.Panic;
14 | import com.northeastern.edu.simpledb.backend.utils.Types;
15 | import com.northeastern.edu.simpledb.common.Error;
16 |
17 | public class DataManger extends AbstractCache implements DataMangerHandler {
18 |
19 | TransactionManager tm;
20 | PageCache pageCache;
21 | Logger logger;
22 | PageIndex pageIndex;
23 |
24 | Page firstPage;
25 |
26 | public DataManger(PageCache pageCache, Logger logger, TransactionManager tm) {
27 | super(0);
28 | this.pageCache = pageCache;
29 | this.logger = logger;
30 | this.tm = tm;
31 | this.pageIndex = new PageIndex();
32 | }
33 |
34 | void initL1Page() {
35 | int pageNumber = pageCache.newPage(L1Page.initRaw());
36 | assert pageNumber == 1;
37 | try {
38 | firstPage = pageCache.getPage(pageNumber);
39 | } catch (Exception e) {
40 | Panic.panic(e);
41 | }
42 | pageCache.flushPage(firstPage);
43 | }
44 |
45 | boolean loadCheckFirstPage() {
46 | try {
47 | firstPage = pageCache.getPage(1);
48 | } catch (Exception e) {
49 | Panic.panic(e);
50 | }
51 | return L1Page.checkVc(firstPage);
52 | }
53 |
54 | // initialize page index after DataManger#open
55 | void fillPageIndex() {
56 | int pageNumber = pageCache.getPageNumber();
57 | // iterate over all page number, then build page index
58 | for (int i = 2; i <= pageNumber; i++) {
59 | Page page = null;
60 | try {
61 | page = pageCache.getPage(i);
62 | } catch (Exception e) {
63 | Panic.panic(e);
64 | }
65 | pageIndex.add(pageNumber, SecondaryPage.getFreeSpace(page));
66 | page.release();
67 | }
68 | }
69 |
70 | // generate log for every transaction referring to `xid`
71 | public void logDataItem(long xid, DataItem dataItem) {
72 | byte[] log = Recover.updateLog(xid, dataItem);
73 | logger.log(log);
74 | }
75 |
76 | // build key from uid, then get data item by key
77 | @Override
78 | protected DataItem getForCache(long uid) throws Exception {
79 | short offset = (short) (uid & ((1L << 16) - 1));
80 | uid >>>= 32;
81 | int pageNumber = (int) (uid & ((1L << 32) - 1));
82 | Page page = pageCache.getPage(pageNumber);
83 | return DataItem.parseDataItem(page, offset, this);
84 | }
85 |
86 | @Override
87 | protected void releaseForCache(DataItem dataItem) {
88 | dataItem.page().release();
89 | }
90 |
91 | // read a data by uid
92 | @Override
93 | public DataItem read(long uid) throws Exception {
94 | DataItem dataItem = super.get(uid);
95 | if (!dataItem.isValid()) {
96 | dataItem.release();
97 | return null;
98 | }
99 | return dataItem;
100 | }
101 |
102 | // insert a data through DataManger
103 | @Override
104 | public long insert(long xid, byte[] data) throws Exception {
105 | byte[] raw = DataItem.wrapDataItemRaw(data);
106 | if (raw.length > SecondaryPage.MAX_FREE_SPACE) throw Error.DataTooLargeException;
107 |
108 | // TODO: need to refactor
109 | // step1 find page index
110 | PageInfo pageInfo = null;
111 | for (int i = 0; i < 5; i++) {
112 | pageInfo = pageIndex.select(raw.length);
113 | if (pageInfo != null) {
114 | break;
115 | } else {
116 | // if no page has enough free space, create a new secondary page
117 | int newPageNumber = pageCache.newPage(SecondaryPage.initRaw());
118 | pageIndex.add(newPageNumber, SecondaryPage.MAX_FREE_SPACE);
119 | }
120 | }
121 |
122 | if (pageInfo == null) throw Error.DataBaseBusyException;
123 |
124 | // step2 write to log file
125 | Page page = null;
126 | int freeSpace = 0;
127 |
128 | try {
129 | page = pageCache.getPage(pageInfo.pageNumber);
130 | byte[] log = Recover.insertLog(xid, page, raw);
131 | logger.log(log);
132 |
133 | // step3 write to page, return offset can be used as uid for cache of data item
134 | short offset = SecondaryPage.insert(page, raw);
135 | page.release();
136 | return Types.addressToUid(pageInfo.pageNumber, offset);
137 | } finally {
138 | // step4 put back page index
139 | if (page != null) {
140 | pageIndex.add(pageInfo.pageNumber, SecondaryPage.getFreeSpace(page));
141 | } else {
142 | pageIndex.add(pageInfo.pageNumber, freeSpace);
143 | }
144 | }
145 |
146 | }
147 |
148 | // close cache and logger
149 | @Override
150 | public void close() {
151 | super.close();
152 | logger.close();
153 |
154 | L1Page.setVcClose(firstPage);
155 | firstPage.release();
156 | pageCache.close();
157 | }
158 |
159 | public void releaseDataItem(DataItem dataItem) {
160 | super.release(dataItem.getUid());
161 | }
162 | }
163 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/DataMangerHandler.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCacheHandler;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger;
7 | import com.northeastern.edu.simpledb.backend.dm.page.L1Page;
8 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
9 |
10 | public interface DataMangerHandler {
11 |
12 | DataItem read(long uid) throws Exception;
13 | long insert(long xid, byte[] data) throws Exception;
14 | void close();
15 |
16 | /**
17 | * create page cache, logger, and data manager
18 | * initialize the L1 page
19 | */
20 | static DataManger create(String path, long memory, TransactionManager tm) {
21 | PageCache pageCache = PageCacheHandler.create(path, memory);
22 | Logger logger = Logger.create(path);
23 |
24 | DataManger dataManger = new DataManger(pageCache, logger, tm);
25 | dataManger.initL1Page();
26 | return dataManger;
27 | }
28 |
29 | static DataManger open(String path, long memory, TransactionManager tm) {
30 | PageCache pageCache = PageCacheHandler.open(path, memory);
31 | Logger logger = Logger.open(path);
32 |
33 | DataManger dataManger = new DataManger(pageCache, logger, tm);
34 | if (!dataManger.loadCheckFirstPage()) { // check failed
35 | Recover.recover(tm, logger, pageCache); // recover based on log file
36 | }
37 | dataManger.fillPageIndex(); // initialize page index
38 | L1Page.setVcOpen(dataManger.firstPage); // set flag for valid check
39 | dataManger.pageCache.flushPage(dataManger.firstPage); // flush to disk immediately
40 | return dataManger;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/Recover.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.backend.common.SubArray;
5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
6 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
7 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger;
8 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
9 | import com.northeastern.edu.simpledb.backend.dm.page.SecondaryPage;
10 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
11 | import com.northeastern.edu.simpledb.backend.utils.Panic;
12 | import com.northeastern.edu.simpledb.backend.utils.Parser;
13 |
14 | import java.util.*;
15 |
16 | public class Recover {
17 |
18 | /**
19 | * insert log: [size][checksum][log type][xid][page number][offset][raw]
20 | * update log: [size][checksum][log type][xid][uid][old data][latest data]
21 | */
22 |
23 | private static final byte LOG_TYPE_INSERT = 0;
24 | private static final byte LOG_TYPE_UPDATE = 1;
25 |
26 | private static final int OF_TYPE = 0;
27 | private static final int OF_XID = OF_TYPE + 1;
28 |
29 | private static final int OF_UPDATE_UID = OF_XID + 8;
30 | private static final int OF_UPDATE_RAW = OF_UPDATE_UID + 8;
31 |
32 | private static final int OF_INSERT_PAGE_NUMBER = OF_XID + 8;
33 | private static final int OF_INSERT_OFFSET = OF_INSERT_PAGE_NUMBER + 4;
34 | private static final int OF_INSERT_RAW = OF_INSERT_OFFSET + 2;
35 |
36 | public static byte[] insertLog(long xid, Page page, byte[] raw) {
37 | // [DataType: 1byte][Xid: 8bytes][PageNumber: 4bytes][Offset: 2bytes][Raw]
38 | byte[] logTypeRaw = {LOG_TYPE_INSERT};
39 | byte[] xidRaw = Parser.long2Byte(xid);
40 | byte[] pageNumberRaw = Parser.int2Byte(page.getPageNumber());
41 | byte[] offsetRaw = Parser.short2Byte(SecondaryPage.getFSO(page));
42 | return Bytes.concat(logTypeRaw, xidRaw, pageNumberRaw, offsetRaw, raw);
43 | }
44 |
45 | static class InsertLogInfo {
46 | long xid;
47 | int pageNumber;
48 | short offset;
49 | byte[] raw;
50 | }
51 |
52 | static class UpdateLogInfo {
53 | long xid;
54 | int pageNumber;
55 | short offset;
56 | byte[] oldRaw;
57 | byte[] newRaw;
58 | }
59 |
60 | public static void recover(TransactionManager tm, Logger logger, PageCache pageCache) {
61 | System.out.println("Recovering");
62 |
63 | logger.rewind();
64 | int maxPage = 0;
65 | byte[] log;
66 | while ((log = logger.next()) != null) {
67 | int pageNumber;
68 | if (isInsertLog(log)) {
69 | InsertLogInfo li = parseInsertLog(log);
70 | pageNumber = li.pageNumber;
71 | } else {
72 | UpdateLogInfo xi = parseUpdateLog(log);
73 | pageNumber = xi.pageNumber;
74 | }
75 | if (pageNumber > maxPage) {
76 | maxPage = pageNumber;
77 | }
78 | }
79 |
80 | if (maxPage == 0) maxPage = 1;
81 | pageCache.truncateByPageNumber(maxPage);
82 | System.out.println("Truncate to " + maxPage + " pages.");
83 |
84 | redoTransactions(tm, logger, pageCache);
85 | System.out.println("Redo Transactions Over.");
86 |
87 | undoTransactions(tm, logger, pageCache);
88 | System.out.println("Undo Transactions Over.");
89 |
90 | System.out.println("Recovery Over.");
91 |
92 | }
93 |
94 | // redo transactions based on log file
95 | private static void redoTransactions(TransactionManager tm, Logger logger, PageCache pageCache) {
96 | logger.rewind();
97 | byte[] log;
98 | while ((log = logger.next()) != null) {
99 | if (isInsertLog(log)) {
100 | InsertLogInfo li = parseInsertLog(log);
101 | if (!tm.isActive(li.xid)) {
102 | doInsertLog(pageCache, log, REDO);
103 | }
104 | } else {
105 | UpdateLogInfo xi = parseUpdateLog(log);
106 | if (!tm.isActive(xi.xid)) {
107 | doUpdateLog(pageCache, log, REDO);
108 | }
109 | }
110 | }
111 | }
112 |
113 | // [log type][xid][uid][old data][latest data]
114 | private static UpdateLogInfo parseUpdateLog(byte[] log) {
115 | UpdateLogInfo updateLogInfo = new UpdateLogInfo();
116 | updateLogInfo.xid = Parser.parseLong(Arrays.copyOfRange(log, OF_XID, OF_UPDATE_UID));
117 | // update id: [page number][offset]
118 | long uid = Parser.parseLong(Arrays.copyOfRange(log, OF_UPDATE_UID, OF_UPDATE_RAW));
119 | updateLogInfo.offset = (short) (uid & ((1L << 16) - 1));
120 | uid >>>= 16;
121 | updateLogInfo.pageNumber = (int) (uid & (1L << 32) - 1);
122 | int length = (log.length - OF_INSERT_RAW) / 2;
123 | updateLogInfo.oldRaw = Arrays.copyOfRange(log, OF_UPDATE_RAW, OF_UPDATE_RAW + length);
124 | updateLogInfo.newRaw = Arrays.copyOfRange(log, OF_UPDATE_RAW + length, OF_UPDATE_RAW + 2 * length);
125 | return updateLogInfo;
126 | }
127 |
128 | // [log type][xid][page number][offset][raw]
129 | private static InsertLogInfo parseInsertLog(byte[] log) {
130 | InsertLogInfo insertLogInfo = new InsertLogInfo();
131 | insertLogInfo.xid = Parser.parseLong(Arrays.copyOfRange(log, OF_XID, OF_INSERT_PAGE_NUMBER));
132 | insertLogInfo.pageNumber = Parser.parseInt(Arrays.copyOfRange(log, OF_INSERT_PAGE_NUMBER, OF_INSERT_OFFSET));
133 | insertLogInfo.offset = Parser.parseShort(Arrays.copyOfRange(log, OF_INSERT_OFFSET, OF_INSERT_RAW));
134 | insertLogInfo.raw = Arrays.copyOfRange(log, OF_INSERT_RAW, log.length);
135 | return insertLogInfo;
136 | }
137 |
138 | // determine type of log based on the first byte
139 | private static boolean isInsertLog(byte[] log) {
140 | return log[0] == LOG_TYPE_INSERT;
141 | }
142 |
143 | // update transactions based on log file
144 | private static void undoTransactions(TransactionManager tm, Logger logger, PageCache pageCache) {
145 | HashMap> logCache = new HashMap();
146 | logger.rewind();
147 | byte[] log;
148 | while ((log = logger.next()) != null) {
149 | if (isInsertLog(log)) {
150 | InsertLogInfo li = parseInsertLog(log);
151 | if (tm.isActive(li.xid)) {
152 | if (!logCache.containsKey(li.xid)) logCache.put(li.xid, new ArrayList<>());
153 | logCache.get(li.xid).add(log);
154 | }
155 | } else {
156 | UpdateLogInfo xi = parseUpdateLog(log);
157 | if (tm.isActive(xi.xid)) {
158 | if (!logCache.containsKey(xi.xid)) logCache.put(xi.xid, new ArrayList<>());
159 | logCache.get(xi.xid).add(log);
160 | }
161 | }
162 | }
163 |
164 | // processing undo logs which state is active reversely
165 | for (Map.Entry> entry : logCache.entrySet()) {
166 | List logs = entry.getValue();
167 | for (int i = logs.size() - 1; i >= 0; i--) {
168 | byte[] tmp = logs.get(i);
169 | if (isInsertLog(tmp)) doInsertLog(pageCache, tmp, UNDO);
170 | else doUpdateLog(pageCache, tmp, UNDO);
171 | }
172 | tm.abort(entry.getKey());
173 | }
174 | }
175 |
176 |
177 | private final static int REDO = 0;
178 | private final static int UNDO = 0;
179 |
180 | private static void doUpdateLog(PageCache pageCache, byte[] log, int flag) {
181 | // step1 determine action based on flag
182 | int pageNumber;
183 | short offset;
184 | byte[] raw;
185 |
186 | UpdateLogInfo updateLogInfo = parseUpdateLog(log);
187 | pageNumber = updateLogInfo.pageNumber;
188 | offset = updateLogInfo.offset;
189 |
190 | if (flag == REDO) raw = updateLogInfo.newRaw; // redo update
191 | else raw = updateLogInfo.oldRaw; // undo update
192 |
193 | // step2 get page by page number
194 | Page page = null;
195 | try {
196 | page = pageCache.getPage(pageNumber);
197 | } catch (Exception e) {
198 | Panic.panic(e);
199 | }
200 |
201 | // step3 update page with the help of `recoverUpdate()` from secondary page
202 | assert page != null;
203 | try {
204 | SecondaryPage.recoverUpdate(page, raw, offset);
205 | } finally {
206 | page.release();
207 | }
208 | }
209 |
210 | private static void doInsertLog(PageCache pageCache, byte[] log, int flag) {
211 | // step1 parsing insert log
212 | InsertLogInfo insertLogInfo = parseInsertLog(log);
213 | int pageNumber = insertLogInfo.pageNumber;
214 |
215 | // step2 determine action based on flag
216 | if (flag == REDO) DataItem.setDataItemRawInvalid(insertLogInfo.raw);
217 |
218 | // step3 get page by page number
219 | Page page = null;
220 | try {
221 | page = pageCache.getPage(pageNumber);
222 | } catch (Exception e) {
223 | Panic.panic(e);
224 | }
225 |
226 | // step4 update page with the help of `recoverInsert()` from secondary page
227 | assert page != null;
228 | try {
229 | SecondaryPage.recoverInsert(page, insertLogInfo.raw, insertLogInfo.offset);
230 | } finally {
231 | page.release();
232 | }
233 | }
234 |
235 | // when DataManger log into log file, data item should be wrapped in advance
236 | public static byte[] updateLog(long xid, DataItem dataItem) {
237 | byte[] logTypeRaw = {LOG_TYPE_UPDATE};
238 | byte[] xidRaw = Parser.long2Byte(xid);
239 | byte[] uidRaw = Parser.long2Byte(dataItem.getUid());
240 | byte[] oldRaw = dataItem.getOldRaw();
241 | SubArray raw = dataItem.getRaw();
242 | byte[] newRaw = Arrays.copyOfRange(raw.raw, raw.start, raw.end);
243 | return Bytes.concat(logTypeRaw, xidRaw, uidRaw, oldRaw, newRaw);
244 | }
245 |
246 | }
247 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/cache/PageCache.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.cache;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache;
4 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
5 | import com.northeastern.edu.simpledb.backend.utils.Panic;
6 | import com.northeastern.edu.simpledb.common.Error;
7 |
8 | import java.io.IOException;
9 | import java.io.RandomAccessFile;
10 | import java.nio.ByteBuffer;
11 | import java.nio.channels.FileChannel;
12 | import java.util.concurrent.atomic.AtomicInteger;
13 | import java.util.concurrent.locks.Lock;
14 | import java.util.concurrent.locks.ReentrantLock;
15 |
16 | public class PageCache extends AbstractCache implements PageCacheHandler {
17 |
18 | private static final int MEM_MIN_LIM = 10;
19 |
20 | public static final String DB_SUFFIX = ".db";
21 |
22 | private RandomAccessFile file;
23 |
24 | private FileChannel fc;
25 |
26 | private Lock fileLock;
27 |
28 | private AtomicInteger pageNumbers;
29 |
30 | public PageCache(RandomAccessFile file, FileChannel fileChannel, int maxResource) {
31 | super(maxResource);
32 | if (maxResource < MEM_MIN_LIM) Panic.panic(Error.MemTooSmallException);
33 | long length = 0;
34 | try {
35 | length = file.length();
36 | } catch (Exception e) {
37 | Panic.panic(e);
38 | }
39 | this.file = file;
40 | this.fc = fileChannel;
41 | this.fileLock = new ReentrantLock();
42 | this.pageNumbers = new AtomicInteger((int) length / PAGE_SIZE);
43 | }
44 |
45 | // turn off caching and write back all resources
46 | @Override
47 | public void close() {
48 | super.close();
49 | try {
50 | fc.close();
51 | file.close();
52 | } catch (IOException e) {
53 | Panic.panic(e);
54 | }
55 | }
56 |
57 | /**
58 | * read page from database file based on pageNumber, and
59 | * encapsulate it as Page
60 | */
61 | @Override
62 | protected Page getForCache(long key) {
63 | int pageNumber = (int) key;
64 | long offset = pageOffset(pageNumber);
65 |
66 | ByteBuffer buf = ByteBuffer.allocate(PAGE_SIZE);
67 | fileLock.lock();
68 | try {
69 | fc.position(offset);
70 | fc.read(buf);
71 | } catch (IOException e) {
72 | Panic.panic(e);
73 | } finally {
74 | fileLock.unlock();
75 | }
76 | return new Page(pageNumber, buf.array(), this);
77 | }
78 |
79 | // calculate offset from the beginning based on page number
80 | static long pageOffset(int pageNumber) {
81 | return (long) (pageNumber - 1) * PAGE_SIZE;
82 | }
83 |
84 | // release cache, and write back to disk
85 | @Override
86 | protected void releaseForCache(Page page) {
87 | if (page.isDirty()) {
88 | flushPage(page);
89 | page.setDirty(false);
90 | }
91 | }
92 |
93 | // create a new page based on data, then flush to disk
94 | @Override
95 | public int newPage(byte[] initData) {
96 | int pageNumber = pageNumbers.incrementAndGet();
97 | Page page = new Page(pageNumber, initData, null);
98 | flush(page);
99 | return pageNumber;
100 | }
101 |
102 | // get page from cache by page number
103 | @Override
104 | public Page getPage(int pageNumber) throws Exception {
105 | return get(pageNumber);
106 | }
107 |
108 | // release page from cache
109 | @Override
110 | public void release(Page page) {
111 | release(page.getPageNumber());
112 | }
113 |
114 | // truncate file based on max page number
115 | @Override
116 | public void truncateByPageNumber(int maxPageNumber) {
117 | long size = pageOffset(maxPageNumber + 1);
118 | try {
119 | file.setLength(size);
120 | } catch (IOException e) {
121 | Panic.panic(e);
122 | }
123 | pageNumbers.set(maxPageNumber);
124 | }
125 |
126 | @Override
127 | public int getPageNumber() {
128 | return pageNumbers.intValue();
129 | }
130 |
131 | @Override
132 | public void flushPage(Page page) {
133 | flush(page);
134 | }
135 |
136 | private void flush(Page page) {
137 | int pageNumber = page.getPageNumber();
138 | long offset = pageOffset(pageNumber);
139 |
140 | fileLock.lock();
141 | try {
142 | ByteBuffer buf = ByteBuffer.wrap(page.getData());
143 | fc.position(offset);
144 | fc.write(buf);
145 | } catch (IOException e) {
146 | Panic.panic(e);
147 | } finally {
148 | fileLock.unlock();
149 | }
150 | }
151 | }
152 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/cache/PageCacheHandler.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.cache;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
4 | import com.northeastern.edu.simpledb.backend.utils.Panic;
5 | import com.northeastern.edu.simpledb.common.Error;
6 |
7 | import java.io.File;
8 | import java.io.FileNotFoundException;
9 | import java.io.RandomAccessFile;
10 | import java.nio.channels.FileChannel;
11 |
12 | public interface PageCacheHandler {
13 |
14 | int PAGE_SIZE = 1 << 13;
15 |
16 | int newPage(byte[] initData);
17 | Page getPage(int pageNumber) throws Exception;
18 | void close();
19 | void release(Page page);
20 |
21 | void truncateByPageNumber(int maxPageNumber);
22 | int getPageNumber();
23 | void flushPage(Page pg);
24 |
25 | static PageCache create(String path, long memory) {
26 | File f = new File(path + PageCache.DB_SUFFIX);
27 | try {
28 | if (!f.createNewFile()) {
29 | Panic.panic(Error.FileExistsException);
30 | }
31 | } catch (Exception e) {
32 | Panic.panic(Error.FileCannotRWException);
33 | }
34 |
35 | FileChannel fc = null;
36 | RandomAccessFile raf = null;
37 | try {
38 | raf = new RandomAccessFile(f, "rw");
39 | fc = raf.getChannel();
40 | } catch (FileNotFoundException e) {
41 | Panic.panic(e);
42 | }
43 | return new PageCache(raf, fc, (int) memory / PAGE_SIZE);
44 | }
45 |
46 | static PageCache open(String path, long memory) {
47 | File f = new File(path + PageCache.DB_SUFFIX);
48 | if (!f.exists()) Panic.panic(Error.FileNotExistsException);
49 | if (!f.canRead() || !f.canWrite()) Panic.panic(Error.FileCannotRWException);
50 |
51 | FileChannel fc = null;
52 | RandomAccessFile raf = null;
53 | try {
54 | raf = new RandomAccessFile(f, "rw");
55 | fc = raf.getChannel();
56 | } catch (FileNotFoundException e) {
57 | Panic.panic(e);
58 | }
59 | return new PageCache(raf, fc, (int) memory / PAGE_SIZE);
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/dataItem/AbstractDataItem.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.dataItem;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.backend.common.SubArray;
5 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
6 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler;
7 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
8 | import com.northeastern.edu.simpledb.backend.utils.Parser;
9 | import com.northeastern.edu.simpledb.backend.utils.Types;
10 |
11 | import java.util.Arrays;
12 |
13 | /**
14 | * AbstractDataItem
15 | * encapsulate data as DataItem and provide abstract objects
16 | * to the upper layer through DataManager
17 | */
18 | public abstract class AbstractDataItem {
19 |
20 | abstract SubArray data();
21 |
22 | abstract void before();
23 | abstract void unBefore();
24 | abstract void after(long xid);
25 | abstract void release();
26 |
27 | abstract void lock();
28 | abstract void unlock();
29 | abstract void rLock();
30 | abstract void rUnLock();
31 |
32 | abstract Page page();
33 | abstract long getUid();
34 | abstract byte[] getOldRaw();
35 | abstract SubArray getRaw();
36 |
37 | // wrapping raw to DataItem
38 | public static byte[] wrapDataItemRaw(byte[] raw) {
39 | byte[] valid = new byte[1];
40 | byte[] size = Parser.short2Byte((short) raw.length);
41 | return Bytes.concat(valid, size, raw);
42 | }
43 |
44 | // parsing raw to DataItem starting from the offset of page
45 | public static DataItem parseDataItem(Page page, short offset, DataManger dm) {
46 | byte[] raw = page.getData();
47 | short dataSize = Parser.parseShort(Arrays.copyOfRange(raw, offset + DataItem.OF_SIZE, offset + DataItem.OF_DATA));
48 | short length = (short) (dataSize + DataItem.OF_DATA); // total length of data item
49 | long uid = Types.addressToUid(page.getPageNumber(), offset);
50 | return new DataItem(new SubArray(raw, offset, offset + length), new byte[length], page, uid, dm);
51 | }
52 |
53 | public static void setDataItemRawInvalid(byte[] raw) {
54 | raw[DataItem.OF_VALID] = 0b1;
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/dataItem/DataItem.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.dataItem;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.SubArray;
4 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
5 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
6 |
7 | import java.util.concurrent.locks.Lock;
8 | import java.util.concurrent.locks.ReadWriteLock;
9 | import java.util.concurrent.locks.ReentrantReadWriteLock;
10 |
11 | public class DataItem extends AbstractDataItem {
12 |
13 | static final int OF_VALID = 0;
14 | static final int OF_SIZE = 1;
15 | static final int OF_DATA = 3;
16 |
17 | static final byte VALID_STATE = 0b0;
18 |
19 | private SubArray raw;
20 | private byte[] oldRaw;
21 | private Lock rLock;
22 | private Lock wLock;
23 | private DataManger dm;
24 | private long uid;
25 | private Page page;
26 |
27 | public DataItem(SubArray raw, byte[] oldRaw, Page page, long uid, DataManger dm) {
28 | this.raw = raw;
29 | this.oldRaw = oldRaw;
30 | ReadWriteLock lock = new ReentrantReadWriteLock();
31 | rLock = lock.readLock();
32 | wLock = lock.writeLock();
33 | this.dm = dm;
34 | this.uid = uid;
35 | this.page = page;
36 | }
37 |
38 | public boolean isValid() {
39 | return raw.raw[raw.start + OF_VALID] == VALID_STATE;
40 | }
41 |
42 | /**
43 | * The array returned by this method is data shared. Since
44 | * the array copy in Java will be allocated to a new address,
45 | * SubArray is used for data sharing between threads.
46 | */
47 | @Override
48 | public SubArray data() {
49 | return new SubArray(raw.raw,raw.start + OF_DATA, raw.end);
50 | }
51 |
52 | /**
53 | * When the upper module tries to modify DataItem, it needs
54 | * to follow a certain process: before modification, the before()
55 | * method needs to be called. When you want to undo the modification,
56 | * the unBefore() method is called. After the modification is
57 | * completed, the after() method is called. The entire process
58 | * is mainly to save the previous phase data and log it in time.
59 | * DM will ensure that modifications to DataItem are atomic.
60 | */
61 |
62 | // save the previous phase data
63 | @Override
64 | public void before() {
65 | wLock.lock();
66 | page.setDirty(true);
67 | System.arraycopy(raw.raw, raw.start, oldRaw, 0, oldRaw.length); // latestRaw becomes oldRaw
68 | }
69 |
70 | // undo the previous action
71 | @Override
72 | public void unBefore() {
73 | wLock.lock();
74 | page.setDirty(true);
75 | System.arraycopy(oldRaw, 0, raw.raw, raw.start, oldRaw.length); // oldRaw overwrite latestRaw
76 | }
77 |
78 | // log in time
79 | @Override
80 | public void after(long xid) {
81 | dm.logDataItem(xid, this);
82 | wLock.unlock();
83 | }
84 |
85 | // after using data item, release it from DataManager
86 | @Override
87 | public void release() {
88 | dm.releaseDataItem(this);
89 | }
90 |
91 | @Override
92 | public void lock() {
93 | wLock.lock();
94 | }
95 |
96 | @Override
97 | public void unlock() {
98 | wLock.unlock();
99 | }
100 |
101 | @Override
102 | public void rLock() {
103 | rLock.lock();
104 | }
105 |
106 | @Override
107 | public void rUnLock() {
108 | rLock.unlock();
109 | }
110 |
111 | @Override
112 | public Page page() {
113 | return page;
114 | }
115 |
116 | @Override
117 | public long getUid() {
118 | return uid;
119 | }
120 |
121 | @Override
122 | public byte[] getOldRaw() {
123 | return oldRaw;
124 | }
125 |
126 | @Override
127 | public SubArray getRaw() {
128 | return raw;
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/logger/AbstractLogger.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.logger;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import com.northeastern.edu.simpledb.backend.utils.Parser;
5 | import com.northeastern.edu.simpledb.common.Error;
6 |
7 | import java.io.File;
8 | import java.io.FileNotFoundException;
9 | import java.io.IOException;
10 | import java.io.RandomAccessFile;
11 | import java.nio.ByteBuffer;
12 | import java.nio.channels.FileChannel;
13 |
14 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX;
15 |
16 | abstract class AbstractLogger {
17 | abstract void log(byte[] data);
18 | abstract void truncate(long x) throws IOException;
19 | abstract byte[] next();
20 | abstract void rewind();
21 | abstract void close();
22 |
23 | // create log file based on path, initialize logger
24 | public static Logger create(String path) {
25 | File f = new File(path + LOG_SUFFIX);
26 | try {
27 | if (!f.createNewFile()) {
28 | Panic.panic(Error.FileExistsException);
29 | }
30 | } catch (IOException e) {
31 | throw new RuntimeException(e);
32 | }
33 |
34 | FileChannel fc = null;
35 | RandomAccessFile raf = null;
36 | try {
37 | raf = new RandomAccessFile(f, "rw");
38 | fc = raf.getChannel();
39 | } catch (FileNotFoundException e) {
40 | Panic.panic(e);
41 | }
42 |
43 | ByteBuffer buf = ByteBuffer.wrap(Parser.int2Byte(0));
44 |
45 | try {
46 | fc.position(0);
47 | fc.write(buf);
48 | } catch (IOException e) {
49 | Panic.panic(e);
50 | }
51 |
52 | return new Logger(raf, fc, 0);
53 | }
54 |
55 | // open log file based on path, init, check and format logger
56 | public static Logger open(String path) {
57 | File f = new File(path + LOG_SUFFIX);
58 |
59 | if (!f.exists()) Panic.panic(Error.FileNotExistsException);
60 | if (!f.canRead() || !f.canWrite()) Panic.panic(Error.FileCannotRWException);
61 |
62 | FileChannel fc = null;
63 | RandomAccessFile raf = null;
64 | try {
65 | raf = new RandomAccessFile(f, "rw");
66 | fc = raf.getChannel();
67 | } catch (FileNotFoundException e) {
68 | Panic.panic(e);
69 | }
70 |
71 | Logger logger = new Logger(raf, fc);
72 | logger.init();
73 |
74 | return logger;
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/logger/Logger.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.logger;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.backend.utils.Panic;
5 | import com.northeastern.edu.simpledb.backend.utils.Parser;
6 | import com.northeastern.edu.simpledb.common.Error;
7 |
8 | import java.io.IOException;
9 | import java.io.RandomAccessFile;
10 | import java.nio.ByteBuffer;
11 | import java.nio.channels.FileChannel;
12 | import java.util.Arrays;
13 | import java.util.concurrent.locks.Lock;
14 | import java.util.concurrent.locks.ReentrantLock;
15 |
16 | import static com.northeastern.edu.simpledb.common.Error.BadLogFileException;
17 |
18 | public class Logger extends AbstractLogger{
19 |
20 | // a prime number for calculating check sum
21 | private static final int SEED = 13331;
22 |
23 | // starting position of attribute `size` in a log
24 | private static final int OF_SIZE = 0;
25 |
26 | // starting position of attribute `check sum` in a log
27 | private static final int OF_CHECKSUM = OF_SIZE + 4;
28 |
29 | // starting position of attribute `data` in a log
30 | private static final int OF_DATA = OF_CHECKSUM + 4;
31 |
32 | // standard suffix of log file
33 | public static final String LOG_SUFFIX = ".log";
34 |
35 | private Lock lock;
36 | private RandomAccessFile raf;
37 | private FileChannel fc;
38 |
39 | // the position of the pointer of logger
40 | private long position;
41 |
42 | // the size of log file, it will be updated when `open()` from disk
43 | private long fileSize;
44 |
45 | // the check sum of log file
46 | private int xCheckSum;
47 |
48 | public Logger(RandomAccessFile raf, FileChannel fc) {
49 | this.raf = raf;
50 | this.fc = fc;
51 | this.lock = new ReentrantLock();
52 | }
53 |
54 | public Logger(RandomAccessFile raf, FileChannel fc, int xCheckSum) {
55 | this.raf = raf;
56 | this.fc = fc;
57 | this.xCheckSum = xCheckSum;
58 | this.lock = new ReentrantLock();
59 | }
60 |
61 | /**
62 | * load check sum of log file, update fileSize as same as the
63 | * length of log file, and validate log file using check sum
64 | */
65 | void init() {
66 | long size = 0;
67 | try {
68 | size = raf.length();
69 | } catch (IOException e) {
70 | Panic.panic(e);
71 | }
72 |
73 | if (size < 4) Panic.panic(BadLogFileException);
74 |
75 | // read size from log file
76 | ByteBuffer buf = ByteBuffer.allocate(4);
77 | try {
78 | fc.position(0);
79 | fc.read(buf);
80 | } catch (IOException e) {
81 | Panic.panic(e);
82 | }
83 |
84 | int xChecksum = Parser.parseInt(buf.array());
85 | this.fileSize = size;
86 | this.xCheckSum = xChecksum;
87 |
88 | checkAndRemoveTail();
89 | }
90 |
91 | // validate log file, remove invalid tail
92 | private void checkAndRemoveTail() {
93 | rewind();
94 |
95 | int xCheck = 0;
96 | while (true) {
97 | // get the next log
98 | byte[] log = internNext();
99 | // calculate xChecksum accumulate
100 | if (log == null) break;
101 | xCheck = calChecksum(xCheck, log);
102 | }
103 |
104 | if (xCheck != xCheckSum) Panic.panic(Error.BadLogFileException);
105 |
106 | try {
107 | truncate(position);
108 | } catch (Exception e) {
109 | Panic.panic(e);
110 | }
111 |
112 | try {
113 | raf.seek(position);
114 | } catch (Exception e) {
115 | Panic.panic(e);
116 | }
117 | rewind();
118 | }
119 |
120 | // calculate check sum of log file, then flush to disk
121 | private void updateChecksum(byte[] log) {
122 | this.xCheckSum = calChecksum(this.xCheckSum, log);
123 | ByteBuffer buf = ByteBuffer.wrap(Parser.int2Byte(this.xCheckSum));
124 | try {
125 | fc.position(0);
126 | fc.write(buf);
127 | fc.force(false);
128 | } catch (IOException e) {
129 | Panic.panic(e);
130 | }
131 | }
132 |
133 | // wrap data as a log
134 | private byte[] wrapLog(byte[] data) {
135 | byte[] checksum = Parser.int2Byte(calChecksum(0, data));
136 | byte[] size = Parser.int2Byte(data.length);
137 | return Bytes.concat(size, checksum, data);
138 | }
139 |
140 | /**
141 | * handle bad tail in two cases
142 | * case1: position + OF_DATA > raf.length || position + OF_DATA + data size > raf.length
143 | * case2: check sum1 != check sum2
144 | */
145 | private byte[] internNext() {
146 | if (position + OF_DATA >= fileSize) return null;
147 |
148 | // step1 check size of a log: [size][checksum][data]
149 | ByteBuffer tmp = ByteBuffer.allocate(4);
150 |
151 | try {
152 | fc.position(position);
153 | fc.read(tmp);
154 | } catch (IOException e) {
155 | Panic.panic(e);
156 | }
157 |
158 | int size = Parser.parseInt(tmp.array());
159 | if (position + OF_DATA + size > fileSize) return null;
160 |
161 | // step2 read whole check sum and data, then validate check sum
162 | ByteBuffer buf = ByteBuffer.allocate(OF_DATA + size);
163 |
164 | try {
165 | fc.position(position);
166 | fc.read(buf);
167 | } catch (Exception e) {
168 | Panic.panic(e);
169 | }
170 |
171 | byte[] log = buf.array();
172 | int checksum1 = calChecksum(0, Arrays.copyOfRange(log, OF_DATA, log.length));
173 | int checksum2 = Parser.parseInt(Arrays.copyOfRange(log, OF_CHECKSUM, OF_DATA));
174 | if (checksum1 != checksum2) return null;
175 |
176 | // step3 update position
177 | position += log.length;
178 | return log;
179 | }
180 |
181 |
182 | // calculate check sum based on xCheck and a log
183 | private int calChecksum(int xCheck, byte[] log) {
184 | for (byte b : log) {
185 | xCheck = xCheck * SEED + b;
186 | }
187 | return xCheck;
188 | }
189 |
190 | @Override
191 | public void truncate(long position) throws IOException {
192 | lock.lock();
193 | try {
194 | fc.truncate(position);
195 | } finally {
196 | lock.unlock();
197 | }
198 | }
199 |
200 | // wrap data to log, write a log to log file, then update check sum of log file
201 | @Override
202 | public void log(byte[] data) {
203 | // step1 build log and its format is like `[size][checksum][data]`
204 | byte[] log = wrapLog(data);
205 |
206 | // step2 write log to file
207 | ByteBuffer buf = ByteBuffer.wrap(log);
208 | lock.lock();
209 | try {
210 | fc.position(fc.size());
211 | fc.write(buf);
212 | } catch (IOException e) {
213 | Panic.panic(e);
214 | } finally {
215 | lock.unlock();
216 | }
217 |
218 | // step3 update check sum
219 | updateChecksum(log);
220 | }
221 |
222 | @Override
223 | public void rewind() {
224 | position = 4;
225 | }
226 |
227 | @Override
228 | public byte[] next() {
229 | lock.lock();
230 | try {
231 | byte[] log = internNext();
232 | if (log == null) return null;
233 | return Arrays.copyOfRange(log, OF_DATA, log.length);
234 | } finally {
235 | lock.unlock();
236 | }
237 | }
238 |
239 | @Override
240 | public void close() {
241 | try {
242 | fc.close();
243 | raf.close();
244 | } catch (IOException e) {
245 | Panic.panic(e);
246 | }
247 | }
248 |
249 | public long getPosition() {
250 | return this.position;
251 | }
252 |
253 | public FileChannel getFc() {
254 | return this.fc;
255 | };
256 |
257 | }
258 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/page/AbstractPage.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.page;
2 |
3 | public abstract class AbstractPage {
4 |
5 | abstract void lock();
6 |
7 | abstract void unlock();
8 |
9 | abstract void release();
10 |
11 | abstract void setDirty(boolean dirty);
12 |
13 | abstract boolean isDirty();
14 |
15 | abstract int getPageNumber();
16 |
17 | abstract byte[] getData();
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/page/L1Page.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.page;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 | import com.northeastern.edu.simpledb.backend.utils.RandomUtil;
5 |
6 | import java.util.Arrays;
7 |
8 | /**
9 | * L1 Page
10 | * when db start up, write random bytes from OF_VC to (OF_VC + LEN_VC - 1) byte
11 | * when db close, copy random bytes from (OF_VC + LEN_VC) to (OF_VC + 2 * LEN_VC - 1) byte
12 | * used to determine whether the database was shut down normally in the last time
13 | */
14 | public class L1Page {
15 |
16 | private final static int OF_VC = 100;
17 |
18 | private final static int LEN_VC = 8;
19 |
20 | public static byte[] initRaw() {
21 | byte[] raw = new byte[PageCache.PAGE_SIZE];
22 | setVcOpen(raw);
23 | return raw;
24 | }
25 |
26 | public static void setVcOpen(Page page) {
27 | page.setDirty(true);
28 | setVcOpen(page.getData());
29 | }
30 |
31 | private static void setVcOpen(byte[] raw) {
32 | System.arraycopy(RandomUtil.randomBytes(LEN_VC), 0, raw, OF_VC, LEN_VC);
33 | }
34 |
35 | public static void setVcClose(Page page) {
36 | page.setDirty(true);
37 | setVcClose(page.getData());
38 | }
39 |
40 | private static void setVcClose(byte[] raw) {
41 | System.arraycopy(raw, OF_VC, raw, OF_VC + LEN_VC, LEN_VC);
42 | }
43 |
44 | public static boolean checkVc(Page pg) {
45 | return checkVc(pg.getData());
46 | }
47 |
48 | private static boolean checkVc(byte[] raw) {
49 | return Arrays.equals(Arrays.copyOfRange(raw, OF_VC, OF_VC + LEN_VC), Arrays.copyOfRange(raw, OF_VC + LEN_VC, OF_VC + 2 * LEN_VC));
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/page/Page.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.page;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 |
5 | import java.util.concurrent.locks.Lock;
6 | import java.util.concurrent.locks.ReentrantLock;
7 |
8 | public class Page extends AbstractPage {
9 |
10 | private int pageNumber;
11 | private byte[] data;
12 | private boolean dirty;
13 | private Lock lock;
14 | private PageCache pageCache;
15 |
16 |
17 | public Page(int pageNumber, byte[] data, PageCache pc) {
18 | this.pageNumber = pageNumber;
19 | this.data = data;
20 | this.pageCache = pc;
21 | this.lock = new ReentrantLock();
22 | }
23 |
24 | @Override
25 | public void lock() {
26 | lock.lock();
27 | }
28 |
29 | @Override
30 | public void unlock() {
31 | lock.unlock();
32 | }
33 |
34 | @Override
35 | public void release() {
36 | pageCache.release(this);
37 | }
38 |
39 | @Override
40 | public void setDirty(boolean dirty) {
41 | this.dirty = dirty;
42 | }
43 |
44 | @Override
45 | public boolean isDirty() {
46 | return this.dirty;
47 | }
48 |
49 | @Override
50 | public int getPageNumber() {
51 | return pageNumber;
52 | }
53 |
54 | @Override
55 | public byte[] getData() {
56 | return data;
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/page/SecondaryPage.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.page;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 | import com.northeastern.edu.simpledb.backend.utils.Parser;
5 |
6 | import java.util.Arrays;
7 |
8 | /**
9 | * Secondary Page
10 | * starting with a 2 bytes unsigned number representing the
11 | * offset of the free location in this pge. the rest is the
12 | * actual stored data
13 | */
14 | public class SecondaryPage {
15 |
16 | private static final short OF_FREE = 0;
17 | private static final short OF_DATA = 2;
18 | public static final int MAX_FREE_SPACE = PageCache.PAGE_SIZE - OF_DATA;
19 |
20 | // init a raw data as secondary page [Offset: 2bytes][Data]
21 | public static byte[] initRaw() {
22 | byte[] raw = new byte[PageCache.PAGE_SIZE];
23 | setFSO(raw, OF_DATA);
24 | return raw;
25 | }
26 |
27 | // insert raw into page, return insertion position
28 | public static short insert(Page page, byte[] raw) {
29 | page.setDirty(true);
30 | // get free space offset
31 | short offset = getFSO(page.getData());
32 | System.arraycopy(raw, 0, page.getData(), offset, raw.length);
33 | // update free space offset
34 | setFSO(page.getData(), (short)(offset + raw.length));
35 | return offset;
36 | }
37 |
38 | private static void setFSO(byte[] raw, short ofData) {
39 | System.arraycopy(Parser.short2Byte(ofData), 0, raw, OF_FREE, OF_DATA);
40 | }
41 |
42 | private static short getFSO(byte[] raw) {
43 | return Parser.parseShort(Arrays.copyOfRange(raw, OF_FREE, OF_DATA));
44 | }
45 |
46 | public static short getFSO(Page page) {
47 | return getFSO(page.getData());
48 | }
49 |
50 | // comes in handy when calculating free space for page index
51 | public static int getFreeSpace(Page page) {
52 | return PageCache.PAGE_SIZE - (int) getFSO(page.getData());
53 | }
54 |
55 | // recover insert statement from log file, update FSO of page if needed
56 | public static void recoverInsert(Page page, byte[] raw, short offset) {
57 | page.setDirty(true);
58 | System.arraycopy(raw, 0, page.getData(), offset, raw.length);
59 |
60 | short rawFSO = getFSO(page.getData());
61 | if (rawFSO < offset + raw.length) setFSO(page.getData(), (short) (offset + raw.length));
62 | }
63 |
64 | // recover update statement from log file
65 | public static void recoverUpdate(Page page, byte[] raw, short offset) {
66 | page.setDirty(true);
67 | System.arraycopy(raw, 0, page.getData(), offset, raw.length);
68 | }
69 |
70 | }
71 |
72 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/pageIndex/PageIndex.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.pageIndex;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 |
5 | import java.util.ArrayList;
6 | import java.util.List;
7 | import java.util.concurrent.locks.Lock;
8 | import java.util.concurrent.locks.ReentrantLock;
9 |
10 | /**
11 | * Page Index
12 | * the page index caches the free space of each page. It is
13 | * used to quickly find a page with suitable space when the
14 | * upper module performs an insertion operation without checking
15 | * the information of each page from the disk or cache
16 | */
17 | public class PageIndex {
18 |
19 | private static final int INTERVALS_NO = 40;
20 | private static final int THRESHOLD = PageCache.PAGE_SIZE / INTERVALS_NO;
21 |
22 | private Lock lock;
23 |
24 | /**
25 | * entry in Map
26 | * key: the number of free slot : Integer
27 | * value: all of PageInfo have the number of free slot : List
28 | */
29 | private List[] lists;
30 |
31 | @SuppressWarnings("unchecked")
32 | public PageIndex() {
33 | this.lock = new ReentrantLock();
34 | this.lists = new List[INTERVALS_NO + 1];
35 | for (int i = 0; i < INTERVALS_NO + 1; i++) {
36 | lists[i] = new ArrayList<>();
37 | }
38 | }
39 |
40 |
41 | public void add(int pageNumber, int freeSpace) {
42 | int number = freeSpace / THRESHOLD; // how many free slot this page has?
43 | lists[number].add(new PageInfo(pageNumber, freeSpace));
44 | }
45 |
46 | /**
47 | * calculating how many free slot needed, then
48 | * accessing the next one, having more slot and
49 | * ensuring the data won't across two pages.
50 | */
51 | public PageInfo select(int spaceSize) {
52 | int number = spaceSize / THRESHOLD; // how many free slot need?
53 | if (number < INTERVALS_NO) number++; // rounded up
54 |
55 | // iterate over lists finding the first page doesn't acquire by other thread
56 | while (number <= INTERVALS_NO) {
57 | if (lists[number].size() == 0) {
58 | number++;
59 | continue;
60 | }
61 | if (lists[number].size() > 0) return lists[number].remove(0);
62 | }
63 | return null;
64 | }
65 |
66 |
67 | }
68 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/dm/pageIndex/PageInfo.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.pageIndex;
2 |
3 | public class PageInfo {
4 | public int pageNumber;
5 | public int freeSpace;
6 |
7 | public PageInfo(int pageNumber, int freeSpace) {
8 | this.pageNumber = pageNumber;
9 | this.freeSpace = freeSpace;
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/im/BPlusTree.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.im;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.SubArray;
4 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.utils.Parser;
7 |
8 | import java.util.ArrayList;
9 | import java.util.Arrays;
10 | import java.util.List;
11 | import java.util.concurrent.locks.Lock;
12 | import java.util.concurrent.locks.ReentrantLock;
13 |
14 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID;
15 |
16 | public class BPlusTree {
17 |
18 | private static final int UID_SIZE = 8;
19 | DataManger dm;
20 | long bootUid;
21 | DataItem bootDataItem;
22 | Lock bootLock;
23 |
24 |
25 | /**
26 | * storing uid of insertion of nil root raw, aka rootUid
27 | * return the uid of insertion of uid of insertion of nil root raw, aka bootUid
28 | */
29 | public static long create(DataManger dm) throws Exception {
30 | byte[] rawRoot = Node.newNilRootRaw(); // get nil root raw: [1,0,0,0,0,0,0,0...]
31 | long rootUid = dm.insert(SUPER_XID, rawRoot); // store nil root raw, and return uid of insertion of nil root raw
32 | return dm.insert(SUPER_XID, Parser.long2Byte(rootUid)); // store uid, return uid of insertion of uid of nil root raw
33 | }
34 |
35 | // loading rootUid by bootUid
36 | public static BPlusTree load(long bootUid, DataManger dm) throws Exception { //
37 | DataItem bootDataItem = dm.read(bootUid);
38 | assert bootDataItem != null;
39 | BPlusTree bPlusTree = new BPlusTree();
40 | bPlusTree.bootDataItem = bootDataItem;
41 | bPlusTree.dm = dm;
42 | bPlusTree.bootUid = bootUid;
43 | bPlusTree.bootLock = new ReentrantLock();
44 | return bPlusTree;
45 | }
46 |
47 | // retrieving rootUid from the data of bootDataItem
48 | private long rootUid() {
49 | bootLock.lock();
50 | try {
51 | SubArray subArray = bootDataItem.data();
52 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start, subArray.start + UID_SIZE));
53 | } finally {
54 | bootLock.unlock();
55 | }
56 | }
57 |
58 | /**
59 | * [newRootUid]
60 | * / \
61 | * [rootUid] [uid]
62 | */
63 | private void updateRootUid(long left, long right, long rightKey) throws Exception {
64 | byte[] rootRaw = Node.newRootRaw(left, right, rightKey);
65 | long newRootUid = dm.insert(SUPER_XID, rootRaw);
66 | bootDataItem.before();
67 | SubArray diRaw = bootDataItem.data();
68 | System.arraycopy(Parser.long2Byte(newRootUid), 0, diRaw.raw, diRaw.start, UID_SIZE);
69 | bootDataItem.after(SUPER_XID);
70 | }
71 |
72 | public void insert(long key, long uid) throws Exception {
73 | long rootUid = rootUid();
74 | InsertRes res = insert(rootUid, uid, key);
75 | assert res != null;
76 | if (res.newNode != 0) updateRootUid(rootUid, res.newNode, res.newKey);
77 | }
78 |
79 | class InsertRes {
80 | long newNode;
81 | long newKey;
82 | }
83 |
84 | /**
85 | * insert uid and key into B+ Tree based on nodeUid
86 | */
87 | private InsertRes insert(long nodeUid, long uid, long key) throws Exception {
88 | Node node = Node.loadNode(this, nodeUid);
89 | boolean isLeaf = node.isLeaf();
90 | node.release();
91 |
92 | // in two situations, insertAndSplit will be invoked
93 | InsertRes res = null;
94 | if (isLeaf) {
95 | // 1. insert to a leaf node
96 | res = insertAndSplit(nodeUid, uid, key);
97 | } else {
98 | long next = searchNext(nodeUid, key); // find the first son has larger key than input key
99 | InsertRes insertRes = insert(next, uid, key);
100 | if (insertRes.newNode != 0) {
101 | // 2. after splitting, a new node created, and this need to be inserted into b+tree based on the current node
102 | res = insertAndSplit(nodeUid, insertRes.newNode, insertRes.newKey);
103 | } else {
104 | res = new InsertRes();
105 | }
106 | }
107 | return res;
108 | }
109 |
110 | private InsertRes insertAndSplit(long nodeUid, long uid, long key) throws Exception {
111 | for (;;) {
112 | Node node = Node.loadNode(this, nodeUid);
113 | Node.InsertAndSplitRes insertAndSplitRes = node.insertAndSplit(uid, key);
114 | if (insertAndSplitRes.siblingUid != 0) {
115 | nodeUid = insertAndSplitRes.siblingUid;
116 | } else {
117 | InsertRes insertRes = new InsertRes();
118 | insertRes.newKey = insertAndSplitRes.newKey;
119 | insertRes.newNode = insertAndSplitRes.newSon;
120 | return insertRes;
121 | }
122 | }
123 | }
124 |
125 | // search in node starting from root util reaching leaf
126 | private long searchLeaf(long nodeUid, long key) throws Exception {
127 | Node node = Node.loadNode(this, nodeUid);
128 | // System.out.println("node = " + node); for test purpose
129 | boolean isLeaf = node.isLeaf();
130 | node.release();
131 |
132 | if (isLeaf) {
133 | return nodeUid;
134 | } else {
135 | long next = searchNext(nodeUid, key);
136 | return searchLeaf(next, key);
137 | }
138 | }
139 |
140 |
141 | // search uid of key in the current node, if not found return uid of the sibling
142 | private long searchNext(long nodeUid, long key) throws Exception{
143 | for(;;) {
144 | Node node = Node.loadNode(this, nodeUid);
145 | Node.SearchNextRes res = node.searchNext(key);
146 | node.release();
147 | if (res.uid != 0) return res.uid;
148 | nodeUid = res.siblingUid;
149 | }
150 | }
151 |
152 | // search() -> searchRange() -> searchLeaf() -> searchNext()
153 | public List search(long key) throws Exception {
154 | return searchRange(key, key);
155 | }
156 |
157 | public List searchRange(long leftKey, long rightKey) throws Exception {
158 | long rootUid = rootUid();
159 | long leftUid = searchLeaf(rootUid, leftKey);
160 | List uids = new ArrayList<>();
161 | for (;;) { // searching all the siblings starting from the given leaf node and collecting all uids
162 | Node leaf = Node.loadNode(this, leftUid);
163 | Node.LeafSearchRangeRes res = leaf.leafSearchRangeRes(leftKey, rightKey);
164 | leaf.release();
165 | uids.addAll(res.uids);
166 | if (res.siblingUid == 0) break;
167 | else leftUid = res.siblingUid;
168 | }
169 | return uids;
170 | }
171 |
172 | public void close() {
173 | bootDataItem.release();
174 | }
175 |
176 | }
177 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/im/Node.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.im;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.SubArray;
4 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
5 | import com.northeastern.edu.simpledb.backend.utils.Parser;
6 |
7 | import java.util.ArrayList;
8 | import java.util.Arrays;
9 | import java.util.List;
10 |
11 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID;
12 |
13 | public class Node {
14 |
15 | // Node will be stored as a DataItem
16 | // [Valid][Size][LeafFlag][NumberOfKeys][SiblingUid][Son0][Key0][Son1][Key1]...[SonN][KeyN]
17 |
18 | private static final int SON_SIZE = 8;
19 |
20 | private static final int KEY_SIZE = 8;
21 |
22 | private static final int NUMBER_OF_KEYS_SIZE = 2;
23 |
24 | private static final int SIBLING_UID_SIZE = 8;
25 |
26 | private static final int LEAF_FLAG_SIZE = 1;
27 |
28 | private static final int IS_LEAF_OFFSET = 0;
29 |
30 | private static final int NUMBER_OF_KEYS_OFFSET = IS_LEAF_OFFSET + LEAF_FLAG_SIZE;
31 | private static final int SIBLING_OFFSET = NUMBER_OF_KEYS_OFFSET + NUMBER_OF_KEYS_SIZE;
32 | private static final int NODE_HEADER_SIZE = SIBLING_OFFSET + SIBLING_UID_SIZE;
33 | private static final int BALANCE_NUMBER = 3;
34 |
35 | private static final int NODE_SIZE = NODE_HEADER_SIZE + (SON_SIZE + KEY_SIZE) * (BALANCE_NUMBER * 2 + 2);
36 |
37 | BPlusTree tree;
38 | DataItem dataItem;
39 | SubArray raw;
40 | long uid;
41 |
42 | static void setRawIsLeaf(SubArray raw, boolean isLeaf) {
43 | raw.raw[raw.start + IS_LEAF_OFFSET] = isLeaf ? (byte) 1 : (byte) 0;
44 | }
45 |
46 | static boolean getRawIfLeaf(SubArray raw) {
47 | return raw.raw[raw.start + IS_LEAF_OFFSET] == (byte) 1;
48 | }
49 |
50 | static void setRawNoKeys(SubArray raw, int numberOfKeys) {
51 | System.arraycopy(Parser.short2Byte((short) numberOfKeys), 0, raw.raw, raw.start + NUMBER_OF_KEYS_OFFSET, NUMBER_OF_KEYS_SIZE);
52 | }
53 |
54 | static int getRawNoKeys(SubArray raw) {
55 | return (int) Parser.parseShort(Arrays.copyOfRange(raw.raw, raw.start + NUMBER_OF_KEYS_OFFSET, raw.start + NUMBER_OF_KEYS_OFFSET + NUMBER_OF_KEYS_SIZE));
56 | }
57 |
58 | static void setRawSibling(SubArray raw, long numberOfSiblings) {
59 | System.arraycopy(Parser.long2Byte(numberOfSiblings), 0, raw.raw, raw.start + SIBLING_OFFSET, SIBLING_UID_SIZE);
60 | }
61 |
62 | static long getRawSibling(SubArray raw) {
63 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, raw.start + SIBLING_OFFSET, raw.start + SIBLING_OFFSET + SIBLING_UID_SIZE));
64 | }
65 |
66 | static void setRawKthSon(SubArray raw, long uid, int kth) {
67 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE);
68 | System.arraycopy(Parser.long2Byte(uid), 0, raw.raw, offset, SON_SIZE);
69 | }
70 |
71 | static long getRawKthSon(SubArray raw, int kth) {
72 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE);
73 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, offset, offset + SON_SIZE));
74 | }
75 |
76 | static void setRawKthKey(SubArray raw, long key, int kth) {
77 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE) + SON_SIZE;
78 | System.arraycopy(Parser.long2Byte(key), 0, raw.raw, offset, KEY_SIZE);
79 | }
80 |
81 | static long getRawKthKey(SubArray raw, int kth) {
82 | int offset = raw.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE) + SON_SIZE;
83 | return Parser.parseLong(Arrays.copyOfRange(raw.raw, offset, offset + KEY_SIZE));
84 | }
85 |
86 | static void shiftRawKth(SubArray raw, int kth) {
87 | int begin = raw.start + NODE_HEADER_SIZE + (kth + 1) * (SON_SIZE + KEY_SIZE);
88 | int end = raw.start + NODE_SIZE - 1;
89 | for (int i = end; i >= begin; i--) {
90 | raw.raw[i] = raw.raw[i - (SON_SIZE + KEY_SIZE)];
91 | }
92 | }
93 |
94 | static void copyRawFromKth(SubArray from, SubArray to, int kth) {
95 | int offset = from.start + NODE_HEADER_SIZE + kth * (SON_SIZE + KEY_SIZE);
96 | System.arraycopy(from.raw, offset, to.raw, to.start + NODE_HEADER_SIZE, from.end - offset);
97 | }
98 |
99 | static byte[] newRootRaw(long left, long right, long key) {
100 | SubArray raw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE);
101 |
102 | setRawIsLeaf(raw, false);
103 | setRawNoKeys(raw, 2);
104 | setRawSibling(raw, 0);
105 | setRawKthSon(raw, left, 0);
106 | setRawKthKey(raw, key, 0);
107 | setRawKthSon(raw, right, 1);
108 | setRawKthKey(raw, Long.MAX_VALUE, 1);
109 |
110 | return raw.raw;
111 | }
112 |
113 | static byte[] newNilRootRaw() {
114 | SubArray raw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE);
115 | setRawIsLeaf(raw, true);
116 | setRawNoKeys(raw, 0);
117 | setRawSibling(raw, 0);
118 |
119 | return raw.raw;
120 | }
121 |
122 | static Node loadNode(BPlusTree bTree, long uid) throws Exception {
123 | DataItem dataItem = bTree.dm.read(uid);
124 | assert dataItem != null;
125 | Node node = new Node();
126 | node.tree = bTree;
127 | node.dataItem = dataItem;
128 | node.raw = dataItem.data();
129 | node.uid = uid;
130 | return node;
131 | }
132 |
133 | public void release() {
134 | dataItem.release();
135 | }
136 |
137 | public boolean isLeaf() {
138 | dataItem.rLock();
139 | try {
140 | return getRawIfLeaf(raw);
141 | } finally {
142 | dataItem.rUnLock();
143 | }
144 | }
145 |
146 |
147 | class SearchNextRes {
148 | long uid;
149 | long siblingUid;
150 | }
151 |
152 | public SearchNextRes searchNext(long key) {
153 | dataItem.rLock();
154 | try {
155 | SearchNextRes res = new SearchNextRes();
156 | int noKeys = getRawNoKeys(raw);
157 | for (int i = 0; i < noKeys; i++) {
158 | long ik = getRawKthKey(raw, i);
159 | if (key < ik) {
160 | res.uid = getRawKthSon(raw, i);
161 | res.siblingUid = 0;
162 | return res;
163 | }
164 | }
165 | res.uid = 0;
166 | res.siblingUid = getRawSibling(raw);
167 | return res;
168 | } finally {
169 | dataItem.rUnLock();
170 | }
171 | }
172 |
173 | class LeafSearchRangeRes {
174 | List uids;
175 | long siblingUid;
176 | }
177 |
178 | public LeafSearchRangeRes leafSearchRangeRes(long leftKey, long rightKey) {
179 | dataItem.rLock();
180 | try {
181 | int noKeys = getRawNoKeys(raw);
182 | int kth = 0;
183 | while (kth < noKeys) {
184 | long ik = getRawKthKey(raw, kth);
185 | if (ik >= leftKey) {
186 | break;
187 | }
188 | kth++;
189 | }
190 | List uids = new ArrayList<>();
191 | while (kth < noKeys) {
192 | long ik = getRawKthKey(raw, kth);
193 | if (ik <= rightKey) {
194 | uids.add(getRawKthSon(raw, kth));
195 | kth ++;
196 | } else break;
197 | }
198 | long siblingUid = 0;
199 | if (kth == noKeys) {
200 | siblingUid = getRawSibling(raw);
201 | }
202 | LeafSearchRangeRes res = new LeafSearchRangeRes();
203 | res.uids = uids;
204 | res.siblingUid = siblingUid;
205 | return res;
206 | } finally {
207 | dataItem.rUnLock();
208 | }
209 | }
210 |
211 | class InsertAndSplitRes {
212 | long siblingUid;
213 | long newSon;
214 | long newKey;
215 | }
216 |
217 | public InsertAndSplitRes insertAndSplit(long uid, long key) throws Exception {
218 | boolean success = false;
219 | Exception err = null;
220 | InsertAndSplitRes insertAndSplitRes = new InsertAndSplitRes();
221 |
222 | dataItem.before();
223 | try {
224 | success = insert(uid, key);
225 | if (!success) {
226 | insertAndSplitRes.siblingUid = getRawSibling(raw);
227 | return insertAndSplitRes;
228 | }
229 | if (needSplit()) {
230 | try {
231 | SplitRes split = split();
232 | insertAndSplitRes.newSon = split.newSon;
233 | insertAndSplitRes.newKey = split.newKey;
234 | return insertAndSplitRes;
235 | } catch (Exception e) {
236 | err = e;
237 | throw e;
238 | }
239 | } else return insertAndSplitRes;
240 | } finally {
241 | if (err == null && success) dataItem.after(SUPER_XID);
242 | else dataItem.unBefore();
243 | }
244 | }
245 |
246 | private boolean insert(long uid, long key) {
247 | int noKeys = getRawNoKeys(raw);
248 | int kth = 0;
249 | while(kth < noKeys) {
250 | long ik = getRawKthKey(raw, kth);
251 | if(ik < key) {
252 | kth ++;
253 | } else {
254 | break;
255 | }
256 | }
257 | if(kth == noKeys && getRawSibling(raw) != 0) return false;
258 |
259 | if(getRawIfLeaf(raw)) {
260 | shiftRawKth(raw, kth);
261 | setRawKthKey(raw, key, kth);
262 | setRawKthSon(raw, uid, kth);
263 | setRawNoKeys(raw, noKeys + 1);
264 | } else {
265 | long kk = getRawKthKey(raw, kth);
266 | setRawKthKey(raw, key, kth);
267 | shiftRawKth(raw, kth + 1);
268 | setRawKthKey(raw, kk, kth + 1);
269 | setRawKthSon(raw, uid, kth + 1);
270 | setRawNoKeys(raw, noKeys + 1);
271 | }
272 | return true;
273 | }
274 |
275 | private boolean needSplit() {
276 | return BALANCE_NUMBER * 2 == getRawNoKeys(raw);
277 | }
278 |
279 | class SplitRes {
280 | long newSon;
281 | long newKey;
282 | }
283 |
284 | private SplitRes split() throws Exception{
285 | System.out.println("split...");
286 | SubArray nodeRaw = new SubArray(new byte[NODE_SIZE], 0, NODE_SIZE);
287 | setRawIsLeaf(nodeRaw, getRawIfLeaf(raw));
288 | setRawNoKeys(nodeRaw, BALANCE_NUMBER);
289 | setRawSibling(nodeRaw, getRawSibling(raw));
290 | copyRawFromKth(raw, nodeRaw, BALANCE_NUMBER);
291 | long son = tree.dm.insert(SUPER_XID, nodeRaw.raw);
292 | setRawNoKeys(raw, BALANCE_NUMBER);
293 | setRawSibling(raw, son);
294 |
295 | SplitRes splitRes = new SplitRes();
296 | splitRes.newSon = son;
297 | splitRes.newKey = getRawKthKey(nodeRaw, 0);
298 | return splitRes;
299 | }
300 |
301 | @Override
302 | public String toString() {
303 | StringBuilder sb = new StringBuilder();
304 | sb.append("Is leaf: ").append(getRawIfLeaf(raw)).append("\n");
305 | int KeyNumber = getRawNoKeys(raw);
306 | sb.append("KeyNumber: ").append(KeyNumber).append("\n");
307 | sb.append("sibling: ").append(getRawSibling(raw)).append("\n");
308 | for(int i = 0; i < KeyNumber; i ++) {
309 | sb.append("son: ").append(getRawKthSon(raw, i)).append(", key: ").append(getRawKthKey(raw, i)).append("\n");
310 | }
311 | return sb.toString();
312 | }
313 | }
314 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/Tokenizer.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser;
2 |
3 | import com.northeastern.edu.simpledb.common.Error;
4 |
5 | //public class Tokenizer {
6 | // private enum State {
7 | // INIT, SYMBOL, QUOTE, TOKEN
8 | // }
9 | //
10 | // private byte[] stat;
11 | // private int pos;
12 | // private String currentToken;
13 | // private boolean flushToken;
14 | // private Exception err;
15 | // private State currentState;
16 | //
17 | // public Tokenizer(byte[] stat) {
18 | // this.stat = stat;
19 | // this.pos = 0;
20 | // this.currentToken = "";
21 | // this.flushToken = true;
22 | // this.currentState = State.INIT;
23 | // }
24 | //
25 | // public String peek() throws Exception {
26 | // if (err != null) {
27 | // throw err;
28 | // }
29 | // if (flushToken) {
30 | // String token = null;
31 | // try {
32 | // token = next();
33 | // } catch (Exception e) {
34 | // err = e;
35 | // throw e;
36 | // }
37 | // currentToken = token;
38 | // flushToken = false;
39 | // }
40 | // return currentToken;
41 | // }
42 | //
43 | // public void pop() {
44 | // flushToken = true;
45 | // }
46 | //
47 | // public byte[] errStat() {
48 | // byte[] res = new byte[stat.length + 3];
49 | // System.arraycopy(stat, 0, res, 0, pos);
50 | // System.arraycopy("<< ".getBytes(), 0, res, pos, 3);
51 | // System.arraycopy(stat, pos, res, pos + 3, stat.length - pos);
52 | // return res;
53 | // }
54 | //
55 | // private void popByte() {
56 | // pos++;
57 | // if (pos > stat.length) {
58 | // pos = stat.length;
59 | // }
60 | // }
61 | //
62 | // private Byte peekByte() {
63 | // if (pos == stat.length) {
64 | // return null;
65 | // }
66 | // return stat[pos];
67 | // }
68 | //
69 | // private String next() throws Exception {
70 | // if (err != null) {
71 | // throw err;
72 | // }
73 | //
74 | // while (true) {
75 | // switch (currentState) {
76 | // case INIT:
77 | // currentState = nextState();
78 | // break;
79 | // case SYMBOL:
80 | // return nextSymbolState();
81 | // case QUOTE:
82 | // return nextQuoteState();
83 | // case TOKEN:
84 | // return nextTokenState();
85 | // }
86 | // }
87 | // }
88 | //
89 | // private State nextState() throws Exception {
90 | // Byte b = peekByte();
91 | // if (b == null) {
92 | // return State.INIT;
93 | // } else if (isSymbol(b)) {
94 | // return State.SYMBOL;
95 | // } else if (b == '"' || b == '\'') {
96 | // return State.QUOTE;
97 | // } else if (isAlphaBeta(b) || isDigit(b)) {
98 | // return State.TOKEN;
99 | // } else {
100 | // err = Error.InvalidCommandException;
101 | // throw err;
102 | // }
103 | // }
104 | //
105 | // private String nextSymbolState() {
106 | // Byte b = peekByte();
107 | // popByte();
108 | // return new String(new byte[]{b});
109 | // }
110 | //
111 | // private String nextTokenState() {
112 | // StringBuilder sb = new StringBuilder();
113 | // while (true) {
114 | // Byte b = peekByte();
115 | // if (b == null || !(isAlphaBeta(b) || isDigit(b) || b == '_')) {
116 | // if (b != null && isBlank(b)) {
117 | // popByte();
118 | // }
119 | // return sb.toString();
120 | // }
121 | // sb.append(new String(new byte[]{b}));
122 | // popByte();
123 | // }
124 | // }
125 | //
126 | // private String nextQuoteState() throws Exception {
127 | // byte quote = peekByte();
128 | // popByte();
129 | // StringBuilder sb = new StringBuilder();
130 | // while (true) {
131 | // Byte b = peekByte();
132 | // if (b == null) {
133 | // err = Error.InvalidCommandException;
134 | // throw err;
135 | // }
136 | // if (b == quote) {
137 | // popByte();
138 | // break;
139 | // }
140 | // sb.append(new String(new byte[]{b}));
141 | // popByte();
142 | // }
143 | // return sb.toString();
144 | // }
145 | //
146 | // private static boolean isDigit(byte b) {
147 | // return (b >= '0' && b <= '9');
148 | // }
149 | //
150 | // private static boolean isAlphaBeta(byte b) {
151 | // return ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'));
152 | // }
153 | //
154 | // private static boolean isSymbol(byte b) {
155 | // return (b == '>' || b == '<' || b == '=' || b == '*' ||
156 | // b == ',' || b == '(' || b == ')');
157 | // }
158 | //
159 | // private static boolean isBlank(byte b) {
160 | // return (b == '\n' || b == ' ' || b == '\t');
161 | // }
162 | //}
163 |
164 | public class Tokenizer {
165 | private enum State {
166 | INIT, IN_SYMBOL, IN_QUOTE, IN_TOKEN, END
167 | }
168 |
169 | private enum TokenType {
170 | SYMBOL, QUOTE, TOKEN
171 | }
172 |
173 | private byte[] stat;
174 | private int pos;
175 | private String currentToken;
176 | private boolean flushToken;
177 | private Exception err;
178 | private State currentState;
179 |
180 | public Tokenizer(byte[] stat) {
181 | this.stat = stat;
182 | this.pos = 0;
183 | this.currentToken = "";
184 | this.flushToken = true;
185 | this.currentState = State.INIT;
186 | }
187 |
188 | public String peek() throws Exception {
189 | if (err != null) {
190 | throw err;
191 | }
192 | if (flushToken) {
193 | String token = null;
194 | try {
195 | token = next();
196 | } catch (Exception e) {
197 | err = e;
198 | throw e;
199 | }
200 | currentToken = token;
201 | flushToken = false;
202 | }
203 | return currentToken;
204 | }
205 |
206 | public void pop() {
207 | flushToken = true;
208 | }
209 |
210 | public byte[] errStat() {
211 | byte[] res = new byte[stat.length + 3];
212 | System.arraycopy(stat, 0, res, 0, pos);
213 | System.arraycopy("<< ".getBytes(), 0, res, pos, 3);
214 | System.arraycopy(stat, pos, res, pos + 3, stat.length - pos);
215 | return res;
216 | }
217 |
218 | private void popByte() {
219 | pos++;
220 | if (pos > stat.length) {
221 | pos = stat.length;
222 | }
223 | }
224 |
225 | private Byte peekByte() {
226 | if (pos == stat.length) {
227 | return null;
228 | }
229 | return stat[pos];
230 | }
231 |
232 | private String next() throws Exception {
233 | if (err != null) {
234 | throw err;
235 | }
236 |
237 | while (true) {
238 | switch (currentState) {
239 | case INIT:
240 | currentState = nextState();
241 | break;
242 | case IN_SYMBOL:
243 | return nextSymbolState();
244 | case IN_QUOTE:
245 | return nextQuoteState();
246 | case IN_TOKEN:
247 | return nextTokenState();
248 | case END:
249 | return "";
250 | }
251 | }
252 | }
253 |
254 | private State nextState() throws Exception {
255 | while(true) {
256 | Byte b = peekByte();
257 | if (b == null) return State.END;
258 | if(!isBlank(b)) {
259 | break;
260 | }
261 | popByte();
262 | }
263 |
264 | Byte b = peekByte();
265 | if (b == null) {
266 | return State.INIT;
267 | } else if (isSymbol(b)) {
268 | return State.IN_SYMBOL;
269 | } else if (b == '"' || b == '\'') {
270 | return State.IN_QUOTE;
271 | } else if (isAlphaBeta(b) || isDigit(b)) {
272 | return State.IN_TOKEN;
273 | } else {
274 | err = Error.InvalidCommandException;
275 | throw err;
276 | }
277 | }
278 |
279 | private String nextSymbolState() {
280 | Byte b = peekByte();
281 | popByte();
282 | this.currentState = State.INIT;
283 | return new String(new byte[]{b});
284 | }
285 |
286 | private String nextTokenState() {
287 | StringBuilder sb = new StringBuilder();
288 | while (true) {
289 | Byte b = peekByte();
290 | if (b == null || !(isAlphaBeta(b) || isDigit(b) || b == '_')) {
291 | if (b != null && isBlank(b)) {
292 | popByte();
293 | }
294 | this.currentState = State.INIT;
295 | return sb.toString();
296 | }
297 | sb.append(new String(new byte[]{b}));
298 | popByte();
299 | }
300 | }
301 |
302 | private String nextQuoteState() throws Exception {
303 | byte quote = peekByte();
304 | popByte();
305 | StringBuilder sb = new StringBuilder();
306 | while (true) {
307 | Byte b = peekByte();
308 | if (b == null) {
309 | err = Error.InvalidCommandException;
310 | throw err;
311 | }
312 | if (b == quote) {
313 | popByte();
314 | break;
315 | }
316 | sb.append(new String(new byte[]{b}));
317 | popByte();
318 | }
319 | this.currentState = State.INIT;
320 | return sb.toString();
321 | }
322 |
323 | private static boolean isDigit(byte b) {
324 | return (b >= '0' && b <= '9');
325 | }
326 |
327 | static boolean isAlphaBeta(byte b) {
328 | return ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'));
329 | }
330 |
331 | private static boolean isSymbol(byte b) {
332 | return (b == '>' || b == '<' || b == '=' || b == '*' ||
333 | b == ',' || b == '(' || b == ')');
334 | }
335 |
336 | private static boolean isBlank(byte b) {
337 | return (b == '\n' || b == ' ' || b == '\t');
338 | }
339 | }
340 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Abort.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Abort {
4 |
5 | }
6 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Begin.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Begin {
4 | public boolean isRepeatableRead;
5 | }
6 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Commit.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Commit {
4 | }
5 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Create.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Create {
4 | public String tableName;
5 | public String[] fieldName;
6 | public String[] fieldType;
7 | public String[] index;
8 | }
9 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Delete.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Delete {
4 | public String tableName;
5 | public Where where;
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Drop.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Drop {
4 | public String tableName;
5 | }
6 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Insert.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Insert {
4 | public String tableName;
5 | public String[] values;
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Select.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Select {
4 | public String tableName;
5 | public String[] fields;
6 | public Where where;
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Show.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Show {
4 | }
5 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/SingleExpression.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class SingleExpression {
4 | public String field;
5 | public String compareOp;
6 | public String value;
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Update.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Update {
4 | public String tableName;
5 | public String fieldName;
6 | public String value;
7 | public Where where;
8 | }
9 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/parser/statement/Where.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser.statement;
2 |
3 | public class Where {
4 | public SingleExpression singleExp1;
5 | public String logicOp;
6 | public SingleExpression singleExp2;
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/server/Executor.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.server;
2 |
3 | import com.northeastern.edu.simpledb.backend.parser.Parser;
4 | import com.northeastern.edu.simpledb.backend.parser.statement.*;
5 | import com.northeastern.edu.simpledb.backend.tbm.BeginRes;
6 | import com.northeastern.edu.simpledb.backend.tbm.TableManager;
7 | import com.northeastern.edu.simpledb.common.Error;
8 |
9 | public class Executor {
10 |
11 | private long xid;
12 | private TableManager tbm;
13 |
14 | public Executor(TableManager tbm) {
15 | this.tbm = tbm;
16 | this.xid = 0;
17 | }
18 |
19 | public byte[] executeTx(byte[] sql) throws Exception {
20 | System.out.println("sql = " + new String(sql));
21 | Object statement = Parser.Parse(sql);
22 | if(statement instanceof Begin) {
23 | if(xid != 0) {
24 | throw Error.NestedTransactionException;
25 | }
26 | BeginRes r = tbm.begin((Begin)statement);
27 | xid = r.xid;
28 | return r.result;
29 | } else if(statement instanceof Commit) {
30 | if(xid == 0) {
31 | throw Error.NoTransactionException;
32 | }
33 | byte[] res = tbm.commit(xid);
34 | xid = 0;
35 | return res;
36 | } else if(statement instanceof Abort) {
37 | if(xid == 0) {
38 | throw Error.NoTransactionException;
39 | }
40 | byte[] res = tbm.abort(xid);
41 | xid = 0;
42 | return res;
43 | } else {
44 | return executeDDLAndDML(statement);
45 | }
46 | }
47 |
48 | private byte[] executeDDLAndDML(Object statement) throws Exception {
49 | boolean tmpTransaction = false;
50 | Exception e = null;
51 | if(xid == 0) {
52 | tmpTransaction = true;
53 | BeginRes r = tbm.begin(new Begin());
54 | xid = r.xid;
55 | }
56 | try {
57 | byte[] res = null;
58 | if(statement instanceof Show) {
59 | res = tbm.show(xid);
60 | } else if(statement instanceof Create) {
61 | res = tbm.create(xid, (Create) statement);
62 | } else if(statement instanceof Select) {
63 | res = tbm.read(xid, (Select) statement);
64 | } else if(statement instanceof Insert) {
65 | res = tbm.insert(xid, (Insert) statement);
66 | } else if(statement instanceof Delete) {
67 | res = tbm.delete(xid, (Delete) statement);
68 | } else if(statement instanceof Update) {
69 | res = tbm.update(xid, (Update) statement);
70 | }
71 | return res;
72 | } catch(Exception ex) {
73 | e = ex;
74 | throw e;
75 | } finally {
76 | if(tmpTransaction) {
77 | if(e != null) {
78 | tbm.abort(xid);
79 | } else {
80 | tbm.commit(xid);
81 | }
82 | xid = 0;
83 | }
84 | }
85 | }
86 |
87 | public void close() {
88 | if (xid != 0) {
89 | System.out.println("Abnormal Abort: " + xid);
90 | tbm.abort(xid);
91 | }
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/server/Server.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.server;
2 |
3 | import com.northeastern.edu.simpledb.backend.tbm.TableManager;
4 | import com.northeastern.edu.simpledb.transport.Encoder;
5 | import com.northeastern.edu.simpledb.transport.Package;
6 | import com.northeastern.edu.simpledb.transport.Packager;
7 | import com.northeastern.edu.simpledb.transport.Transporter;
8 |
9 | import java.io.IOException;
10 | import java.net.InetSocketAddress;
11 | import java.net.ServerSocket;
12 | import java.net.Socket;
13 | import java.util.concurrent.ArrayBlockingQueue;
14 | import java.util.concurrent.ThreadPoolExecutor;
15 | import java.util.concurrent.TimeUnit;
16 |
17 | public class Server {
18 | private int port;
19 | TableManager tbm;
20 |
21 | public Server(int port, TableManager tbm) {
22 | this.port = port;
23 | this.tbm = tbm;
24 | }
25 |
26 | public void start() {
27 | ServerSocket welcomeSocket = null;
28 |
29 | try {
30 | welcomeSocket = new ServerSocket(port);
31 | } catch (IOException e) {
32 | e.printStackTrace();
33 | return ;
34 | }
35 |
36 | try {
37 | while (true) {
38 | Socket socket = welcomeSocket.accept();
39 |
40 | System.out.println("Server listen to port: " + port);
41 | ThreadPoolExecutor tpe = new ThreadPoolExecutor(10, 20, 1L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), new ThreadPoolExecutor.CallerRunsPolicy());
42 |
43 | Runnable worker = new HandleSocket(socket, tbm);
44 | tpe.submit(worker);
45 | }
46 | } catch (IOException e) {
47 | e.printStackTrace();
48 | } finally {
49 | try {
50 | welcomeSocket.close();
51 | } catch (IOException e) {}
52 | }
53 |
54 | }
55 |
56 | }
57 | class HandleSocket implements Runnable {
58 |
59 | private Socket socket;
60 | private TableManager tbm;
61 |
62 | public HandleSocket(Socket socket, TableManager tbm) {
63 | this.socket = socket;
64 | this.tbm = tbm;
65 | }
66 |
67 | @Override
68 | public void run() {
69 | InetSocketAddress address = (InetSocketAddress) socket.getRemoteSocketAddress();
70 | System.out.println("Establish connection: " + address.getAddress().getHostAddress() + ":" + address.getPort());
71 | Packager packager = null;
72 | try {
73 | Transporter t = new Transporter(socket);
74 | Encoder e = new Encoder();
75 | packager = new Packager(t, e);
76 | } catch (IOException e) {
77 | e.printStackTrace();
78 | try {
79 | socket.close();
80 | } catch (IOException e1) {
81 | e1.printStackTrace();
82 | }
83 | return;
84 | }
85 | Executor executor = new Executor(tbm);
86 | while (true) {
87 | Package pkg = null;
88 | try {
89 | pkg = packager.receive();
90 | } catch (Exception e) {
91 | break;
92 | }
93 | byte[] sql = pkg.getData();
94 | byte[] result = null;
95 | Exception e = null;
96 | try {
97 | result = executor.executeTx(sql);
98 | } catch (Exception ex) {
99 | e = ex;
100 | e.printStackTrace();
101 | }
102 |
103 | pkg = new Package(result, e);
104 | try {
105 | packager.send(pkg);
106 | } catch (IOException ex) {
107 | ex.printStackTrace();
108 | break;
109 | }
110 | }
111 |
112 | executor.close();
113 | try {
114 | packager.close();
115 | } catch (Exception e) {
116 | e.printStackTrace();
117 | }
118 | }
119 | }
120 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/AbstractTableManager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.parser.statement.*;
5 | import com.northeastern.edu.simpledb.backend.utils.Parser;
6 | import com.northeastern.edu.simpledb.backend.vm.VersionManager;
7 |
8 | public abstract class AbstractTableManager {
9 | VersionManager vm;
10 | DataManger dm;
11 | public abstract BeginRes begin(Begin begin);
12 | public abstract byte[] commit(long xid) throws Exception;
13 | public abstract byte[] abort(long xid);
14 | public abstract byte[] show(long xid);
15 | public abstract byte[] create(long xid, Create create) throws Exception;
16 | public abstract byte[] insert(long xid, Insert insert) throws Exception;
17 | public abstract byte[] read(long xid, Select select) throws Exception;
18 | public abstract byte[] update(long xid, Update update) throws Exception;
19 | public abstract byte[] delete(long xid, Delete delete) throws Exception;
20 |
21 | public static TableManager create(String path, VersionManager vm, DataManger dm) {
22 | Booter booter = Booter.create(path);
23 | booter.update(Parser.long2Byte(0));
24 | return new TableManager(vm, dm, booter);
25 | }
26 |
27 | public static TableManager open(String path, VersionManager vm, DataManger dm) {
28 | Booter booter = Booter.open(path);
29 | return new TableManager(vm, dm ,booter);
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/BeginRes.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | public class BeginRes {
4 | public long xid;
5 | public byte[] result;
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/Booter.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import com.northeastern.edu.simpledb.common.Error;
5 |
6 | import java.io.File;
7 | import java.io.FileOutputStream;
8 | import java.io.IOException;
9 | import java.nio.file.Files;
10 | import java.nio.file.StandardCopyOption;
11 |
12 | public class Booter {
13 |
14 | private static final String BOOTER_SUFFIX = ".bt";
15 | private static final String BOOTER_TMP_SUFFIX = ".bt_tmp";
16 |
17 | String path;
18 | File file;
19 |
20 | public Booter(String path, File file) {
21 | this.path = path;
22 | this.file = file;
23 | }
24 |
25 | public static Booter open(String path) {
26 | removeBadTmp(path);
27 | File tmp = new File(path + BOOTER_SUFFIX);
28 | if (!tmp.exists()) {
29 | Panic.panic(Error.FileNotExistsException);
30 | }
31 | if (!tmp.canWrite() || !tmp.canRead()) {
32 | Panic.panic(Error.FileCannotRWException);
33 | }
34 | return new Booter(path, tmp);
35 | }
36 |
37 | public static Booter create(String path) {
38 | removeBadTmp(path);
39 | File tmp = new File(path + BOOTER_SUFFIX);
40 | try {
41 | if (!tmp.createNewFile()) {
42 | Panic.panic(Error.FileExistsException);
43 | }
44 | } catch (IOException e) {
45 | Panic.panic(e);
46 | }
47 | if (!tmp.canRead() || !tmp.canWrite()) {
48 | Panic.panic(Error.FileCannotRWException);
49 | }
50 | return new Booter(path, tmp);
51 | }
52 |
53 | private static void removeBadTmp(String path) {
54 | new File(path + BOOTER_TMP_SUFFIX).delete();
55 | }
56 |
57 | public void update(byte[] data) {
58 | File tmp = new File(path + BOOTER_TMP_SUFFIX);
59 | try {
60 | tmp.createNewFile();
61 | } catch (Exception e) {
62 | Panic.panic(e);
63 | }
64 | if (!tmp.canRead() || !tmp.canWrite()) {
65 | Panic.panic(Error.FileCannotRWException);
66 | }
67 | try (FileOutputStream out = new FileOutputStream(tmp)) {
68 | out.write(data);
69 | out.flush();
70 | } catch (Exception e) {
71 | Panic.panic(e);
72 | }
73 |
74 | try {
75 | Files.move(tmp.toPath(), new File(path + BOOTER_SUFFIX).toPath(), StandardCopyOption.REPLACE_EXISTING);
76 | } catch (IOException e) {
77 | Panic.panic(e);
78 | }
79 |
80 | file = new File(path + BOOTER_SUFFIX);
81 | if (!file.exists()) {
82 | Panic.panic(Error.FileNotExistsException);
83 | }
84 | if(!file.canRead() || !file.canWrite()) {
85 | Panic.panic(Error.FileCannotRWException);
86 | }
87 | }
88 |
89 | public byte[] load() {
90 | byte[] bytes = null;
91 | try {
92 | bytes = Files.readAllBytes(file.toPath());
93 | } catch (IOException e) {
94 | Panic.panic(e);
95 | }
96 | return bytes;
97 | }
98 |
99 |
100 | }
101 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/Field.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.backend.im.BPlusTree;
5 | import com.northeastern.edu.simpledb.backend.parser.statement.SingleExpression;
6 | import com.northeastern.edu.simpledb.backend.utils.Panic;
7 | import com.northeastern.edu.simpledb.backend.utils.ParseStringRes;
8 | import com.northeastern.edu.simpledb.backend.utils.Parser;
9 | import com.northeastern.edu.simpledb.common.Error;
10 |
11 | import java.util.Arrays;
12 | import java.util.List;
13 |
14 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID;
15 |
16 | public class Field {
17 |
18 | String fieldName;
19 | String fieldType;
20 | long index;
21 | long uid;
22 | private Table table;
23 | private BPlusTree bPlusTree;
24 |
25 | public Field(long uid, Table table) {
26 | this.uid = uid;
27 | this.table = table;
28 | }
29 |
30 | public Field(Table table, String fieldName, String fieldType, long index) {
31 | this.fieldName = fieldName;
32 | this.fieldType = fieldType;
33 | this.index = index;
34 | this.table = table;
35 | }
36 |
37 | // given table and uid, return a field
38 | public static Field loadField(Table table, long uid) {
39 | byte[] raw = null;
40 | try {
41 | raw = table.tbm.vm.read(SUPER_XID, uid);
42 | } catch (Exception e) {
43 | Panic.panic(e);
44 | }
45 | assert raw != null;
46 | return new Field(uid, table).parseSelf(raw);
47 | }
48 |
49 | public static Field createField(Table table, long xid, String fieldName, String fieldType, boolean indexed) throws Exception {
50 | typeCheck(fieldType);
51 | Field field = new Field(table, fieldName, fieldType, 0);
52 | // if this field is an index, create a B+ Tree for it
53 | if (indexed) {
54 | long index = BPlusTree.create(table.tbm.dm);
55 | BPlusTree bpt = BPlusTree.load(index, table.tbm.dm);
56 | field.index = index;
57 | field.bPlusTree = bpt;
58 | }
59 | field.persistSelf(xid);
60 | return field;
61 | }
62 |
63 | public void insert(Object key, long uid) throws Exception {
64 | long uKey = value2UKey(key);
65 | bPlusTree.insert(uKey, uid);
66 | }
67 |
68 | /**
69 | * convert object to uid key used for searching or insertion of B+ Tree
70 | * eg: where id = 8, here 8 is input object key, and it will be parsed
71 | * into int32
72 | */
73 | private long value2UKey(Object key) {
74 | long uid = 0;
75 | switch(fieldType) {
76 | case "string":
77 | uid = Parser.str2Uid((String) key);
78 | break;
79 | case "int32":
80 | int uint = (int) key;
81 | return (long) uint;
82 | case "int64":
83 | uid = (long) key;
84 | break;
85 | }
86 | return uid;
87 | }
88 |
89 | // convert object to a byte array for storing into disk
90 | public byte[] value2Raw(Object v) {
91 | byte[] raw = null;
92 | switch(fieldType) {
93 | case "int32":
94 | raw = Parser.int2Byte((int) v);
95 | break;
96 | case "int64":
97 | raw = Parser.long2Byte((long) v);
98 | break;
99 | case "string":
100 | raw = Parser.string2Byte((String) v);
101 | break;
102 | }
103 | return raw;
104 | }
105 |
106 | // convert single expression into FieldCalRes
107 | public FieldCalRes calExp(SingleExpression exp) {
108 | Object v;
109 | FieldCalRes fieldCalRes = new FieldCalRes();
110 | switch (exp.compareOp) {
111 | case "<":
112 | fieldCalRes.left = 0;
113 | v = string2Value(exp.value);
114 | long right = value2UKey(v);
115 | fieldCalRes.right = right;
116 | if (fieldCalRes.right > 0) {
117 | fieldCalRes.right --;
118 | }
119 | break;
120 | case "=":
121 | v = string2Value(exp.value);
122 | fieldCalRes.left = value2UKey(v);
123 | fieldCalRes.right = fieldCalRes.left;
124 | break;
125 | case ">":
126 | fieldCalRes.right = Long.MAX_VALUE;
127 | v = string2Value(exp.value);
128 | fieldCalRes.left = value2UKey(v);
129 | break;
130 | }
131 | return fieldCalRes;
132 | }
133 |
134 |
135 | private Field parseSelf(byte[] raw) {
136 | int position = 0;
137 | ParseStringRes parseStringRes = null;
138 | parseStringRes = Parser.parseString(raw);
139 | position += parseStringRes.next;
140 | fieldName = parseStringRes.str;
141 | parseStringRes = Parser.parseString(Arrays.copyOfRange(raw, position, raw.length));
142 | fieldType = parseStringRes.str;
143 | position += parseStringRes.next;
144 | index = Parser.parseLong(Arrays.copyOfRange(raw, position, position + Long.BYTES));
145 | if (index != 0) {
146 | try {
147 | BPlusTree.load(index, table.tbm.dm);
148 | } catch (Exception e) {
149 | Panic.panic(e);
150 | }
151 | }
152 | return this;
153 | }
154 |
155 | private void persistSelf(long xid) throws Exception {
156 | byte[] nameRaw = Parser.string2Byte(fieldName);
157 | byte[] fieldRaw = Parser.string2Byte(fieldType);
158 | byte[] indexRaw = Parser.long2Byte(index);
159 | this.uid = table.tbm.vm.insert(xid, Bytes.concat(nameRaw, fieldRaw, indexRaw));
160 | }
161 |
162 | public Object string2Value(String str) {
163 | switch(fieldType) {
164 | case "int32":
165 | return Integer.parseInt(str);
166 | case "int64":
167 | return Long.parseLong(str);
168 | case "string":
169 | return str;
170 | }
171 | return null;
172 | }
173 |
174 | private static void typeCheck(String fieldType) throws Exception {
175 | if(!"int32".equals(fieldType) && !"int64".equals(fieldType) && !"string".equals(fieldType)) {
176 | throw Error.InvalidFieldException;
177 | }
178 | }
179 |
180 | public boolean isIndexed() {
181 | return index != 0;
182 | }
183 |
184 | public List search(long l, long r) throws Exception {
185 | return bPlusTree.searchRange(l, r);
186 | }
187 |
188 | public String printValue(Object v) {
189 | String str = null;
190 | switch(fieldType) {
191 | case "int32":
192 | str = String.valueOf((int) v);
193 | break;
194 | case "int64":
195 | str = String.valueOf((long) v);
196 | break;
197 | case "string":
198 | str = (String) v;
199 | break;
200 | }
201 | return str;
202 | }
203 |
204 | class ParseValueRes {
205 | Object v;
206 | int shift;
207 | }
208 |
209 | public ParseValueRes parseValue(byte[] raw) {
210 | ParseValueRes parseValueRes = new ParseValueRes();
211 | switch (fieldType) {
212 | case "int32":
213 | parseValueRes.v = Parser.parseInt(raw);
214 | parseValueRes.shift = Integer.BYTES;
215 | break;
216 | case "int64":
217 | parseValueRes.v = Parser.parseLong(raw);
218 | parseValueRes.shift = Long.BYTES;
219 | break;
220 | case "string":
221 | ParseStringRes parseStringRes = Parser.parseString(raw);
222 | parseValueRes.v = parseStringRes.str;
223 | parseValueRes.shift = parseStringRes.next;
224 | break;
225 | }
226 | return parseValueRes;
227 | }
228 | }
229 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/FieldCalRes.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | public class FieldCalRes {
4 | public long left;
5 | public long right;
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tbm/TableManager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tbm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.parser.statement.*;
5 | import com.northeastern.edu.simpledb.backend.utils.Parser;
6 | import com.northeastern.edu.simpledb.backend.vm.VersionManager;
7 | import com.northeastern.edu.simpledb.common.Error;
8 |
9 | import java.util.ArrayList;
10 | import java.util.List;
11 | import java.util.Map;
12 | import java.util.concurrent.ConcurrentHashMap;
13 | import java.util.concurrent.locks.Lock;
14 | import java.util.concurrent.locks.ReentrantLock;
15 |
16 | public class TableManager extends AbstractTableManager{
17 |
18 | private Booter booter;
19 | private Map tableCache;
20 | private Map> xidTableCache;
21 | private Lock lock;
22 |
23 | public TableManager(VersionManager vm, DataManger dm, Booter booter) {
24 | this.booter = booter;
25 | this.vm = vm;
26 | this.dm = dm;
27 | this.tableCache = new ConcurrentHashMap<>();
28 | this.xidTableCache = new ConcurrentHashMap<>();
29 | lock = new ReentrantLock();
30 | loadTables();
31 | }
32 |
33 | private void loadTables() {
34 | long uid = firstTableUid();
35 | while (uid != 0) {
36 | Table table = Table.loadTable(this, uid);
37 | uid = table.nextUid;
38 | tableCache.put(table.name, table);
39 | }
40 | }
41 |
42 | private long firstTableUid() {
43 | byte[] raw = booter.load();
44 | return Parser.parseLong(raw);
45 | }
46 |
47 | private void updateFirstTableUid(long uid) {
48 | byte[] raw = Parser.long2Byte(uid);
49 | booter.update(raw);
50 | }
51 |
52 | @Override
53 | public BeginRes begin(Begin begin) {
54 | BeginRes beginRes = new BeginRes();
55 | int isolationLevel = begin.isRepeatableRead ? 1 : 0;
56 | beginRes.xid = vm.begin(isolationLevel);
57 | beginRes.result = "begin".getBytes();
58 | return beginRes;
59 | }
60 |
61 | @Override
62 | public byte[] commit(long xid) throws Exception {
63 | vm.commit(xid);
64 | return "commit".getBytes();
65 | }
66 |
67 | @Override
68 | public byte[] abort(long xid) {
69 | vm.abort(xid);
70 | return "abort".getBytes();
71 | }
72 |
73 | @Override
74 | public byte[] show(long xid) {
75 | lock.lock();
76 | try {
77 | StringBuilder sb = new StringBuilder();
78 | for (Table tb : tableCache.values()) {
79 | sb.append(tb.toString()).append("\n");
80 | }
81 | List t = xidTableCache.get(xid);
82 | if(t == null) {
83 | return "\n".getBytes();
84 | }
85 | for (Table tb : t) {
86 | sb.append(tb.toString()).append("\n");
87 | }
88 | return sb.toString().getBytes();
89 | } finally {
90 | lock.unlock();
91 | }
92 | }
93 |
94 | @Override
95 | public byte[] create(long xid, Create create) throws Exception {
96 | lock.lock();
97 | try {
98 | if (tableCache.containsKey(create.tableName)) throw Error.DuplicatedTableException;
99 | Table table = Table.createTable(this, firstTableUid(), xid, create); // How does it work?
100 | updateFirstTableUid(table.uid);
101 | tableCache.put(create.tableName, table);
102 | xidTableCache.computeIfAbsent(xid, k -> new ArrayList<>()).add(table);
103 | return ("create " + create.tableName).getBytes();
104 | } finally {
105 | lock.unlock();
106 | }
107 | }
108 |
109 | @Override
110 | public byte[] insert(long xid, Insert insert) throws Exception {
111 | Table table = tableCache.get(insert.tableName);
112 | if (table == null) throw Error.TableNotFoundException;
113 | table.insert(xid, insert);
114 | return "insert".getBytes();
115 | }
116 |
117 | @Override
118 | public byte[] read(long xid, Select read) throws Exception {
119 | Table table = tableCache.get(read.tableName);
120 | if (table == null) throw Error.TableNotFoundException;
121 | return table.read(xid, read).getBytes();
122 | }
123 |
124 | @Override
125 | public byte[] update(long xid, Update update) throws Exception {
126 | Table table = tableCache.get(update.tableName);
127 | if(table == null) throw Error.TableNotFoundException;
128 | int count = table.update(xid, update);
129 | return ("update " + count).getBytes();
130 | }
131 |
132 | @Override
133 | public byte[] delete(long xid, Delete delete) throws Exception {
134 | Table table = tableCache.get(delete.tableName);
135 | if(table == null) throw Error.TableNotFoundException;
136 | int count = table.delete(xid, delete);
137 | return ("delete " + count).getBytes();
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tm/AbstractTransactionManager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tm;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import com.northeastern.edu.simpledb.common.Error;
5 |
6 | import java.io.File;
7 | import java.io.FileNotFoundException;
8 | import java.io.IOException;
9 | import java.io.RandomAccessFile;
10 | import java.nio.ByteBuffer;
11 | import java.nio.channels.FileChannel;
12 |
13 | public abstract class AbstractTransactionManager {
14 | abstract long begin();
15 | abstract void commit(long xid);
16 | abstract void abort(long xid);
17 | abstract boolean isActive(long xid);
18 | abstract boolean isCommitted(long xid);
19 | abstract boolean isAborted(long xid);
20 | abstract void close();
21 |
22 | public static TransactionManager create(String path) {
23 | File f = new File(path + TransactionManager.XID_SUFFIX);
24 | try {
25 | if (!f.createNewFile()) {
26 | Panic.panic(Error.BadXIDFileException);
27 | }
28 | } catch (IOException e) {
29 | Panic.panic(e);
30 | }
31 |
32 | RandomAccessFile raf = null;
33 | FileChannel fc = null;
34 | try {
35 | raf = new RandomAccessFile(f, "rw");
36 | fc = raf.getChannel();
37 | } catch (IOException e) {
38 | Panic.panic(e);
39 | }
40 |
41 | ByteBuffer buf = ByteBuffer.allocate(TransactionManager.XID_HEADER_LENGTH);
42 | try {
43 | fc.position(0);
44 | fc.write(buf);
45 | } catch (IOException e) {
46 | Panic.panic(e);
47 | }
48 |
49 | return new TransactionManager(raf, fc);
50 | }
51 |
52 | public static TransactionManager open(String path) {
53 | File f = new File(path + TransactionManager.XID_SUFFIX);
54 | if(!f.exists()) {
55 | Panic.panic(Error.FileNotExistsException);
56 | }
57 | if(!f.canRead() || !f.canWrite()) {
58 | Panic.panic(Error.FileCannotRWException);
59 | }
60 | FileChannel fc = null;
61 | RandomAccessFile raf = null;
62 | try {
63 | raf = new RandomAccessFile(f, "rw");
64 | fc = raf.getChannel();
65 | } catch (FileNotFoundException e) {
66 | Panic.panic(e);
67 | }
68 |
69 | return new TransactionManager(raf, fc);
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/tm/TransactionManager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tm;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import com.northeastern.edu.simpledb.backend.utils.Parser;
5 | import com.northeastern.edu.simpledb.common.Error;
6 |
7 | import java.io.IOException;
8 | import java.io.RandomAccessFile;
9 | import java.nio.ByteBuffer;
10 | import java.nio.channels.FileChannel;
11 | import java.util.concurrent.locks.Lock;
12 | import java.util.concurrent.locks.ReentrantLock;
13 |
14 |
15 | /*
16 |
17 | */
18 | public class TransactionManager extends AbstractTransactionManager{
19 |
20 | // the length of header of xid file, unit is byte
21 | static final int XID_HEADER_LENGTH = 8;
22 |
23 | // the size of each xid, unit is byte
24 | private static final int XID_FIELD_SIZE = 1;
25 |
26 | // three states of transaction
27 | private static final byte FIELD_TRAN_ACTIVE = 0;
28 | private static final byte FIELD_TRAN_COMMITTED = 1;
29 | private static final byte FIELD_TRAN_ABORTED = 2;
30 |
31 | // super transaction whose state is always committed
32 | public static final long SUPER_XID = 0;
33 |
34 | // standard suffix of xid file
35 | public static final String XID_SUFFIX = ".xid";
36 |
37 | private RandomAccessFile file;
38 | private FileChannel fc;
39 | private long xidCounter;
40 | private Lock counterLock;
41 |
42 | public TransactionManager(RandomAccessFile raf, FileChannel fc) {
43 | this.file = raf;
44 | this.fc = fc;
45 | this.counterLock = new ReentrantLock();
46 | checkXIDCounter();
47 | }
48 |
49 | /**
50 | * check if the xid file is valid
51 | * read the xidcounter in XID_FILE_HEADER, calculate
52 | * the theoretical length of the file based on it, and
53 | * compare it with the actual length
54 | */
55 | private void checkXIDCounter() {
56 | long fileLength = 0;
57 | try {
58 | fileLength = file.length();
59 | } catch (IOException e) {
60 | Panic.panic(Error.BadXIDFileException);
61 | }
62 | if (fileLength < XID_HEADER_LENGTH) {
63 | Panic.panic(Error.BadXIDFileException);
64 | }
65 |
66 | ByteBuffer buf = ByteBuffer.allocate(XID_HEADER_LENGTH);
67 | try {
68 | fc.position(0);
69 | fc.read(buf);
70 | } catch (IOException e) {
71 | Panic.panic(e);
72 | }
73 | this.xidCounter = Parser.parseLong(buf.array());
74 | long end = getXidPosition(this.xidCounter + 1);
75 | if (end != fileLength) {
76 | Panic.panic(Error.BadXIDFileException);
77 | }
78 | }
79 |
80 | /**
81 | * calculate its position in xid file based on input `xid`
82 | * `xid` means it is the normal xid transaction starting from 1
83 | * the xid of super transaction is 0
84 | */
85 | private long getXidPosition(long xid) {
86 | if (xid == 0) {
87 | throw new RuntimeException("position of xid can't be zero");
88 | }
89 | return (xid - 1) * XID_FIELD_SIZE + XID_HEADER_LENGTH;
90 | }
91 |
92 | // update state by xid
93 | private void updateXID(long xid, byte state) {
94 | long offset = getXidPosition(xid);
95 | byte[] tmp = new byte[XID_FIELD_SIZE];
96 | tmp[0] = state;
97 | try {
98 | fc.position(offset);
99 | fc.write(ByteBuffer.wrap(tmp));
100 | } catch (IOException e) {
101 | Panic.panic(e);
102 | }
103 |
104 | flushToDisk();
105 | }
106 |
107 | // increase one to xid counter and update header of xid file
108 | private void incrXIDCounter() {
109 | xidCounter++;
110 | ByteBuffer buf = ByteBuffer.wrap(Parser.long2Byte(xidCounter));
111 | try {
112 | fc.position(0);
113 | fc.write(buf);
114 | } catch (IOException e) {
115 | Panic.panic(e);
116 | }
117 |
118 | flushToDisk();
119 | }
120 |
121 | private void flushToDisk() {
122 | try {
123 | fc.force(false);
124 | } catch (IOException e) {
125 | Panic.panic(e);
126 | }
127 | }
128 |
129 | // start a new transaction
130 | @Override
131 | public long begin() {
132 | counterLock.lock();
133 | try {
134 | long xid = xidCounter + 1;
135 | updateXID(xid, FIELD_TRAN_ACTIVE);
136 | incrXIDCounter();
137 | return xid;
138 | } finally {
139 | counterLock.unlock();
140 | }
141 | }
142 |
143 | // commit a transaction
144 | @Override
145 | public void commit(long xid) {
146 | updateXID(xid, FIELD_TRAN_COMMITTED);
147 | }
148 |
149 | // abort a transaction
150 | @Override
151 | public void abort(long xid) {
152 | updateXID(xid, FIELD_TRAN_ABORTED);
153 | }
154 |
155 | // check if xid transaction is in input `state`
156 | private boolean checkXID(long xid, byte state) {
157 | long offset = getXidPosition(xid);
158 | ByteBuffer buf = ByteBuffer.allocate(XID_FIELD_SIZE);
159 | try {
160 | fc.position(offset);
161 | fc.read(buf);
162 | } catch (IOException e) {
163 | Panic.panic(e);
164 | }
165 | return buf.array()[0] == state;
166 | }
167 |
168 | @Override
169 | public boolean isActive(long xid) {
170 | if(xid == SUPER_XID) return false;
171 | return checkXID(xid, FIELD_TRAN_ACTIVE);
172 | }
173 |
174 | @Override
175 | public boolean isCommitted(long xid) {
176 | if(xid == SUPER_XID) return true;
177 | return checkXID(xid, FIELD_TRAN_COMMITTED);
178 | }
179 |
180 | @Override
181 | public boolean isAborted(long xid) {
182 | if(xid == SUPER_XID) return false;
183 | return checkXID(xid, FIELD_TRAN_ABORTED);
184 | }
185 |
186 | @Override
187 | public void close() {
188 | try {
189 | fc.close();
190 | file.close();
191 | } catch (IOException e) {
192 | Panic.panic(e);
193 | }
194 | }
195 | }
196 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/utils/Panic.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.utils;
2 |
3 | public class Panic {
4 | public static void panic(Exception err) {
5 | err.printStackTrace();
6 | System.exit(1);
7 | }
8 | }
9 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/utils/ParseStringRes.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.utils;
2 |
3 | public class ParseStringRes {
4 | public String str;
5 | public int next;
6 |
7 | public ParseStringRes(String str, int next) {
8 | this.str = str;
9 | this.next = next;
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/utils/Parser.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.utils;
2 |
3 | import com.google.common.primitives.Bytes;
4 |
5 | import java.nio.ByteBuffer;
6 | import java.nio.charset.StandardCharsets;
7 | import java.util.Arrays;
8 |
9 | public class Parser {
10 |
11 | public static long parseLong(byte[] bytes) {
12 | ByteBuffer buf = ByteBuffer.wrap(bytes, 0, Long.BYTES);
13 | return buf.getLong();
14 | }
15 |
16 | public static byte[] long2Byte(long value) {
17 | return ByteBuffer.allocate(Long.BYTES).putLong(value).array();
18 | }
19 |
20 | public static short parseShort(byte[] bytes) {
21 | ByteBuffer buf = ByteBuffer.wrap(bytes, 0, Short.BYTES);
22 | return buf.getShort();
23 | }
24 |
25 | public static byte[] short2Byte(short value) {
26 | return ByteBuffer.allocate(Short.BYTES).putShort(value).array();
27 | }
28 |
29 | public static byte[] int2Byte(int value) {
30 | return ByteBuffer.allocate(Integer.BYTES).putInt(value).array();
31 | }
32 |
33 | public static int parseInt(byte[] raw) {
34 | return ByteBuffer.wrap(raw, 0, Integer.BYTES).getInt();
35 | }
36 |
37 | public static ParseStringRes parseString(byte[] raw) {
38 | int length = parseInt(Arrays.copyOf(raw, Integer.BYTES));
39 | String str = new String(Arrays.copyOfRange(raw, Integer.BYTES, Integer.BYTES + length));
40 | return new ParseStringRes(str, length + Integer.BYTES);
41 | }
42 |
43 | public static byte[] string2Byte(String str) {
44 | byte[] length = int2Byte(str.length());
45 | return Bytes.concat(length, str.getBytes(StandardCharsets.UTF_8));
46 | }
47 |
48 | public static long str2Uid(String key) {
49 | long seed = 13331;
50 | long res = 0;
51 | for(byte b : key.getBytes()) {
52 | res = res * seed + (long)b;
53 | }
54 | return res;
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/utils/RandomUtil.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.utils;
2 |
3 | import java.security.SecureRandom;
4 |
5 | public class RandomUtil {
6 |
7 | public static byte[] randomBytes(int length) {
8 | SecureRandom random = new SecureRandom();
9 | byte[] bytes = new byte[length];
10 | random.nextBytes(bytes);
11 | return bytes;
12 | }
13 |
14 | }
15 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/utils/Types.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.utils;
2 |
3 | public class Types {
4 | public static long addressToUid(int pageNumber, short offset) {
5 | long u0 = pageNumber;
6 | long u1 = offset;
7 | return u0 << 32 | u1;
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/Entry.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.backend.common.SubArray;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.utils.Parser;
7 |
8 | import java.util.Arrays;
9 |
10 | /**
11 | * VM provide Entry to the upper layer
12 | * [XMIN][XMAX][data]
13 | */
14 | public class Entry {
15 |
16 | private static final int OF_XMIN = 0;
17 | private static final int OF_XMAX = OF_XMIN + 8;
18 | private static final int OF_DATA = OF_XMAX + 8;
19 |
20 | private long uid;
21 | private DataItem dataItem;
22 | private VersionManager vm;
23 | private static Entry newEntry(VersionManager vm, DataItem di, long uid) {
24 | Entry entry = new Entry();
25 | entry.uid = uid;
26 | entry.dataItem = di;
27 | entry.vm = vm;
28 | return entry;
29 | }
30 |
31 | public static Entry loadEntry(VersionManager vm, long uid) throws Exception {
32 | DataItem di = vm.dm.read(uid);
33 | return newEntry(vm, di, uid);
34 | }
35 |
36 | public static byte[] wrapEntryRaw(long xid, byte[] data) {
37 | byte[] xmin = Parser.long2Byte(xid);
38 | byte[] xmax = new byte[8];
39 | return Bytes.concat(xmin, xmax, data);
40 | }
41 |
42 | public void release() {
43 | vm.releaseEntry(this);
44 | }
45 |
46 | public byte[] data() {
47 | dataItem.rLock();
48 | try {
49 | SubArray subArray = dataItem.data();
50 | byte[] data = new byte[subArray.end - (subArray.start + OF_DATA)];
51 | System.arraycopy(subArray.raw, subArray.start + OF_DATA, data, 0, data.length);
52 | return data;
53 | } finally {
54 | dataItem.rUnLock();
55 | }
56 | }
57 |
58 | public long getXmin() {
59 | dataItem.rLock();
60 | try {
61 | SubArray subArray = dataItem.data();
62 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start + OF_XMIN, subArray.start + OF_XMAX));
63 | } finally {
64 | dataItem.rUnLock();
65 | }
66 | }
67 |
68 | public long getXmax() {
69 | dataItem.rLock();
70 | try {
71 | SubArray subArray = dataItem.data();
72 | return Parser.parseLong(Arrays.copyOfRange(subArray.raw, subArray.start + OF_XMAX, subArray.start + OF_DATA));
73 | } finally {
74 | dataItem.rUnLock();
75 | }
76 | }
77 |
78 | public void setXmax(long xid) {
79 | dataItem.before();
80 | try {
81 | SubArray subArray = dataItem.data();
82 | System.arraycopy(Parser.long2Byte(xid), 0, subArray.raw, subArray.start + OF_XMAX, 8);
83 | } finally {
84 | dataItem.after(xid);
85 | }
86 | }
87 |
88 | public long getUid() {
89 | return uid;
90 | }
91 |
92 | public void remove() {
93 | dataItem.release();
94 | }
95 |
96 |
97 | }
98 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/LockTable.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import java.util.*;
4 | import java.util.concurrent.locks.Lock;
5 | import java.util.concurrent.locks.ReentrantLock;
6 |
7 | import com.northeastern.edu.simpledb.common.Error;
8 | import com.northeastern.edu.simpledb.common.Error.*;
9 |
10 | public class LockTable {
11 |
12 | /**
13 | * what uids that xid is holding
14 | * format: {xid: [uid0, uid1, uid2...]}
15 | */
16 | private Map> x2u;
17 |
18 | /**
19 | * what xid is holding the uid
20 | * format: {uid: xid}
21 | */
22 | private Map u2x;
23 |
24 | /**
25 | * all of xid are waiting uid
26 | * format: {uid: [xid0, xid1, xid2...]}
27 | */
28 | private Map> wait;
29 |
30 | /**
31 | * the lock of xid is waiting resource
32 | * format: {xid: Lock}
33 | */
34 | private Map waitLock;
35 |
36 | /**
37 | * what uid that xid is waiting for
38 | * format: {xid : uid}
39 | */
40 | private Map waitU;
41 |
42 | private Lock lock;
43 |
44 | public LockTable() {
45 | x2u = new HashMap<>();
46 | u2x = new HashMap<>();
47 | wait = new HashMap<>();
48 | waitLock = new HashMap<>();
49 | waitU = new HashMap<>();
50 | lock = new ReentrantLock();
51 | }
52 |
53 | // add a relationship which is xid is waiting uid
54 | public Lock add(long xid, long uid) throws Exception{
55 | lock.lock();
56 | try {
57 | // xid is holding uid
58 | if (isInList(x2u, xid, uid)) return null;
59 |
60 | // if no one is holding uid, add the relationship directly (acquired resource)
61 | if (!u2x.containsKey(uid)) {
62 | u2x.put(uid, xid);
63 | putIntoList(x2u, xid, uid);
64 | return null;
65 | }
66 |
67 | // other is holding uid
68 | // try to wait
69 | waitU.put(xid, uid);
70 | putIntoList(wait, uid, xid);
71 |
72 | // check deadlock
73 | if (hasDeadLock()) {
74 | waitU.remove(xid);
75 | removeFromList(wait, uid, xid);
76 | throw Error.DeadLockException;
77 | }
78 |
79 | // no deadlock, start waiting
80 | Lock l = new ReentrantLock();
81 | l.lock();
82 | waitLock.put(xid, l); // store lock for the releasing later
83 | return l;
84 | } finally {
85 | lock.unlock();
86 | }
87 | }
88 |
89 | // remove a relationship between xid and uid
90 | public void remove(long xid) {
91 | lock.lock();
92 | try {
93 | List uids = x2u.get(xid);
94 | if (uids != null) {
95 | while (uids.size() > 0) {
96 | Long uid = uids.remove(0);
97 | selectNewXID(uid); // select the next xid for uid from candidates
98 | }
99 | }
100 | } finally {
101 | lock.unlock();
102 | }
103 | }
104 |
105 | // select the next xid for uid from candidates
106 | private void selectNewXID(Long uid) {
107 | u2x.remove(uid); // remove the relationship which is uid is acquired by xid
108 | List xids = wait.get(uid); // get candidates queue
109 | if (xids == null) return ;
110 | assert xids.size() > 0;
111 |
112 | while (xids.size() > 0) {
113 | Long xid = xids.remove(0);
114 | if (!waitLock.containsKey(xid)) { // no lock found for the xid
115 | continue;
116 | } else { // assign uid this xid
117 | u2x.put(uid, xid);
118 | putIntoList(x2u, xid, uid);
119 | Lock l = waitLock.remove(xid);
120 | waitU.remove(xid);
121 | l.unlock();
122 | }
123 | }
124 | }
125 |
126 | private void removeFromList(Map> wait, long uid, long xid) {
127 | List list = wait.getOrDefault(uid, List.of());
128 | Iterator iterator = list.iterator();
129 | while (iterator.hasNext()) {
130 | Long e;
131 | if ((e = iterator.next()) == xid) {
132 | iterator.remove();
133 | break;
134 | }
135 | }
136 | if (list.size() == 0) wait.remove(uid);
137 | }
138 |
139 | private void putIntoList(Map> map, long key, long value) {
140 | map.computeIfAbsent(key, k -> new ArrayList<>()).add(0, value);
141 | }
142 |
143 | private boolean isInList(Map> x2u, long xid, long uid) {
144 | return x2u.getOrDefault(xid, List.of()).stream().anyMatch(e -> e == uid);
145 | }
146 |
147 |
148 | private Map xidStamp;
149 | private int stamp;
150 |
151 | private boolean hasDeadLock() {
152 | xidStamp = new HashMap<>();
153 | stamp = 1;
154 | // iterate x2u
155 | for (long xid : x2u.keySet()) {
156 | Integer s = xidStamp.get(xid);
157 | if (s != null && s > 0) continue;
158 | stamp++;
159 | if(dfs(xid)) {
160 | return true;
161 | }
162 | }
163 | return false;
164 | }
165 |
166 | /**
167 | * see what uid that the xid is waiting for, and
168 | * check if another xid holding the uid is waiting
169 | * the uid acquired by the xid
170 | */
171 | private boolean dfs(long xid) {
172 | Integer stp = xidStamp.get(xid);
173 | if(stp != null && stp == stamp) {
174 | return true;
175 | }
176 | if(stp != null && stp < stamp) {
177 | return false;
178 | }
179 | xidStamp.put(xid, stamp);
180 |
181 | Long uid = waitU.get(xid); // what uid that this xid is waiting for
182 | if(uid == null) return false;
183 | Long x = u2x.get(uid); // what xid is holding this uid
184 | assert x != null;
185 | return dfs(x);
186 | }
187 | }
188 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/Transaction.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID;
7 |
8 | public class Transaction {
9 | public long xid;
10 | public int level;
11 | public Map snapshot;
12 | public Exception err;
13 | public boolean autoAborted;
14 |
15 | public static Transaction newTransaction(long xid, int level, Map active) {
16 | Transaction transaction = new Transaction();
17 | transaction.xid = xid;
18 | transaction.level = level;
19 | if (level != 0) {
20 | transaction.snapshot = new HashMap<>();
21 | for (Long x : active.keySet()) {
22 | transaction.snapshot.put(x, true);
23 | }
24 | }
25 | return transaction;
26 | }
27 |
28 | public boolean isInSnapShot(long xid) {
29 | if (xid == SUPER_XID) return false;
30 | return snapshot.containsKey(xid);
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/VersionManager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.AbstractCache;
4 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
5 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
6 | import com.northeastern.edu.simpledb.backend.utils.Panic;
7 | import com.northeastern.edu.simpledb.common.Error;
8 |
9 | import java.util.HashMap;
10 | import java.util.Map;
11 | import java.util.concurrent.ConcurrentHashMap;
12 | import java.util.concurrent.locks.Lock;
13 | import java.util.concurrent.locks.ReentrantLock;
14 |
15 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.SUPER_XID;
16 | import static com.northeastern.edu.simpledb.common.Error.NullEntryException;
17 |
18 | public class VersionManager extends AbstractCache implements VersionManagerHandler {
19 |
20 | TransactionManager tm;
21 | DataManger dm;
22 | Map activeTransaction;
23 | Lock lock;
24 | LockTable lt;
25 |
26 | public VersionManager(TransactionManager tm, DataManger dm) {
27 | super(0);
28 | this.tm = tm;
29 | this.dm = dm;
30 | activeTransaction = new ConcurrentHashMap<>();
31 | activeTransaction.put(SUPER_XID, Transaction.newTransaction(SUPER_XID, 0, null));
32 | this.lock = new ReentrantLock();
33 | this.lt = new LockTable();
34 | }
35 |
36 | @Override
37 | public byte[] read(long xid, long uid) throws Exception {
38 | Transaction transaction = activeTransaction.get(xid);
39 | if (transaction == null) return null;
40 | if (transaction.err != null) throw transaction.err;
41 |
42 | Entry entry = null;
43 | try {
44 | entry = super.get(uid);
45 | } catch (Exception e) {
46 | if (e == NullEntryException) {
47 | return null;
48 | } else throw e;
49 | }
50 |
51 | try {
52 | if (Visibility.isVisible(tm, transaction, entry)) {
53 | return entry.data();
54 | } else return null;
55 | } finally {
56 | entry.release();
57 | }
58 | }
59 |
60 | @Override
61 | public long insert(long xid, byte[] data) throws Exception {
62 | Transaction transaction = activeTransaction.get(xid);
63 | if (transaction == null) throw Error.InvalidCommandException;
64 | if (transaction.err != null) throw transaction.err;
65 |
66 | byte[] raw = Entry.wrapEntryRaw(xid, data);
67 | return dm.insert(xid, raw);
68 | }
69 |
70 | @Override
71 | public boolean delete(long xid, long uid) throws Exception {
72 | Transaction transaction = activeTransaction.get(xid);
73 | if (transaction == null) throw Error.InvalidCommandException;
74 | if (transaction.err != null) throw transaction.err;
75 |
76 |
77 | Entry entry = null;
78 | try {
79 | entry = super.get(uid);
80 | } catch (Exception e) {
81 | if (e == NullEntryException) {
82 | return false;
83 | } else throw e;
84 | }
85 |
86 | try {
87 | if (!Visibility.isVisible(tm, transaction, entry)) {
88 | return false;
89 | }
90 |
91 | Lock l = null;
92 | try {
93 | l = lt.add(xid, uid); // try to delete it
94 | } catch (Exception e) {
95 | transaction.err = Error.ConcurrentUpdateException;
96 | internAbort(xid, true);
97 | transaction.autoAborted = true;
98 | throw transaction.err;
99 | }
100 |
101 | if (l != null) {
102 | l.lock();
103 | l.unlock();
104 | }
105 |
106 | if (entry.getXmax() == xid) { // has been deleted
107 | return false;
108 | }
109 |
110 | if (Visibility.isVersionSkip(tm, transaction, entry)) { // version skip, do rollback
111 | transaction.err = Error.ConcurrentUpdateException;
112 | internAbort(xid, true);
113 | transaction.autoAborted = true;
114 | throw transaction.err;
115 | }
116 |
117 | entry.setXmax(xid); // signal entry as deleted
118 | return true;
119 |
120 | } finally {
121 | entry.release();
122 | }
123 | }
124 |
125 | /**
126 | * abort delete when exception happens
127 | * @param autoAborted: if it happens by accident, autoAborted is true
128 | */
129 | private void internAbort(long xid, boolean autoAborted) {
130 | lock.lock();
131 | Transaction transaction = activeTransaction.get(xid);
132 | if (!autoAborted) activeTransaction.remove(xid);
133 | lock.unlock();
134 |
135 | if (!transaction.autoAborted) {
136 | lt.remove(xid);
137 | tm.abort(xid);
138 | }
139 | }
140 |
141 | /**
142 | * start a transaction
143 | * when new a transaction, do a snapshot of active transaction
144 | * for the validation of visibility
145 | */
146 | @Override
147 | public long begin(int level) {
148 | lock.lock();
149 | try {
150 | long xid = tm.begin();
151 | Transaction transaction = Transaction.newTransaction(xid, level, activeTransaction);
152 | activeTransaction.put(xid, transaction);
153 | return xid;
154 | } finally {
155 | lock.unlock();
156 | }
157 | }
158 |
159 | @Override
160 | public void commit(long xid) throws Exception {
161 | Transaction transaction = activeTransaction.get(xid);
162 |
163 | try {
164 | if (transaction.err != null) throw transaction.err;
165 | } catch (NullPointerException e) {
166 | System.out.println(xid);
167 | System.out.println(activeTransaction.keySet());
168 | Panic.panic(e);
169 | }
170 |
171 | activeTransaction.remove(xid);
172 |
173 | lt.remove(xid);
174 | tm.commit(xid);
175 | }
176 |
177 | @Override
178 | public void abort(long xid) {
179 | internAbort(xid, false);
180 | }
181 |
182 | @Override
183 | protected Entry getForCache(long uid) throws Exception {
184 | Entry entry = Entry.loadEntry(this, uid);
185 | if (entry == null) throw NullEntryException;
186 | return entry;
187 | }
188 |
189 | @Override
190 | protected void releaseForCache(Entry entry) {
191 | entry.remove();
192 | }
193 |
194 | public void releaseEntry(Entry entry) {
195 | super.release(entry.getUid());
196 | }
197 | }
198 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/VersionManagerHandler.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
5 |
6 | public interface VersionManagerHandler {
7 | byte[] read(long xid, long uid) throws Exception;
8 | long insert(long xid, byte[] data) throws Exception;
9 | boolean delete(long xid, long uid) throws Exception;
10 |
11 | long begin(int level);
12 | void commit(long xid) throws Exception;
13 | void abort(long xid);
14 |
15 | static VersionManagerHandler newVersionManager(TransactionManager tm, DataManger dm) {
16 | return new VersionManager(tm, dm);
17 | }
18 |
19 |
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/backend/vm/Visibility.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
4 |
5 | public class Visibility {
6 |
7 | /**
8 | * version skip happens when e has been updated by t1 from v0
9 | * to v1, but t1 can't be seen by t2, and t2 try to update e,
10 | * the version of it will transfer from v0 to v2, skipping v1
11 | */
12 | public static boolean isVersionSkip(TransactionManager tm, Transaction t, Entry e) {
13 | long xmax = e.getXmax();
14 | if (t.level == 0) return false;
15 | else return tm.isCommitted(xmax) && (xmax > t.xid || t.isInSnapShot(xmax));
16 | }
17 |
18 | // determine if record(e) is visible to transaction(t) based on isolation level
19 | protected static boolean isVisible(TransactionManager tm, Transaction t, Entry e) {
20 | if(t.level == 0) return readCommitted(tm, t, e);
21 | else return repeatableRead(tm, t, e);
22 | }
23 |
24 | private static boolean readCommitted(TransactionManager tm, Transaction t, Entry e) {
25 | long xid = t.xid;
26 | long xmin = e.getXmin();
27 | long xmax = e.getXmax();
28 | if (xmin == xid && xmax == 0) return true; // e had been created by the current transaction and wasn't deleted
29 |
30 | if (tm.isCommitted(xmin)) {
31 | if (xmax == 0) return true; // e had been committed and wasn't deleted
32 | return xmax != xid && !tm.isCommitted(xmax); // e had been deleted by other transaction but wasn't committed
33 | }
34 | return false;
35 | }
36 |
37 | private static boolean repeatableRead(TransactionManager tm, Transaction t, Entry e) {
38 | long xid = t.xid;
39 | long xmin = e.getXmin();
40 | long xmax = e.getXmax();
41 | if (xmin == xid && xmax == 0) return true; // e had been created by the current transaction and wasn't deleted
42 |
43 | if (tm.isCommitted(xmin) && xmin < xid && !t.isInSnapShot(xmin)) { // e was committed before when the current transaction created
44 | if (xmax == 0) return true; // not deleted so far
45 | if (xmax != xid) {
46 | return !tm.isCommitted(xmax) || xmax > xid || t.isInSnapShot(xmax); // e was deleted by the transaction created after the current transaction
47 | }
48 | }
49 | return false;
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/client/Client.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.client;
2 |
3 | import com.northeastern.edu.simpledb.transport.Package;
4 | import com.northeastern.edu.simpledb.transport.Packager;
5 |
6 | public class Client {
7 | private RoundTripper rt;
8 |
9 | public Client(Packager packager) {
10 | this.rt = new RoundTripper(packager);
11 | }
12 |
13 | public byte[] execute(byte[] stat) throws Exception {
14 | Package pkg = new Package(stat, null);
15 | Package resPkg = rt.roundTrip(pkg);
16 | if (resPkg.getErr() != null) {
17 | throw resPkg.getErr();
18 | }
19 | return resPkg.getData();
20 | }
21 |
22 | public void close() {
23 | try {
24 | rt.close();
25 | } catch (Exception e) {}
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/client/Launcher.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.client;
2 |
3 | import com.northeastern.edu.simpledb.transport.Encoder;
4 | import com.northeastern.edu.simpledb.transport.Packager;
5 | import com.northeastern.edu.simpledb.transport.Transporter;
6 |
7 | import java.io.IOException;
8 | import java.net.InetAddress;
9 | import java.net.Socket;
10 | import java.net.UnknownHostException;
11 |
12 | public class Launcher {
13 | public static void main(String[] args) throws IOException {
14 | Socket socket = new Socket(InetAddress.getLocalHost().getHostAddress(), 9999);
15 | Encoder encoder = new Encoder();
16 | Transporter transporter = new Transporter(socket);
17 | Packager packager = new Packager(transporter, encoder);
18 |
19 | Client client = new Client(packager);
20 | Shell shell = new Shell(client);
21 | shell.run();
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/client/RoundTripper.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.client;
2 |
3 | import com.northeastern.edu.simpledb.transport.Package;
4 | import com.northeastern.edu.simpledb.transport.Packager;
5 |
6 | import java.io.IOException;
7 |
8 | public class RoundTripper {
9 |
10 | private Packager packager;
11 |
12 | public RoundTripper(Packager packager) {
13 | this.packager = packager;
14 | }
15 |
16 | public Package roundTrip(Package pkg) throws Exception {
17 | packager.send(pkg);
18 | return packager.receive();
19 | }
20 |
21 | public void close() throws IOException {
22 | packager.close();
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/client/Shell.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.client;
2 |
3 | import java.util.Scanner;
4 |
5 | public class Shell {
6 |
7 | private Client client;
8 |
9 | public Shell(Client client) {
10 | this.client = client;
11 | }
12 |
13 | public void run() {
14 | Scanner scanner = new Scanner(System.in);
15 | try {
16 | while (true) {
17 | System.out.print(":> ");
18 | String statStr = scanner.nextLine().strip();
19 | if ("exit".equals(statStr) || "quit".equals(statStr)) break;
20 | try {
21 | byte[] res = client.execute(statStr.getBytes());
22 | System.out.println(new String(res));
23 | } catch (Exception e) {
24 | System.out.println(e.getMessage());
25 | }
26 | }
27 | } finally {
28 | scanner.close();
29 | client.close();
30 | }
31 | }
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/common/Error.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.common;
2 |
3 | public class Error {
4 |
5 | // common
6 | public static final Exception FileNotExistsException = new RuntimeException("File does not exists!");
7 | public static final Exception FileExistsException = new RuntimeException("File already exists!");
8 | public static final Exception FileCannotRWException = new RuntimeException("File cannot read or write!");
9 |
10 | public static final Exception CacheFullException = new RuntimeException("Cache is full!");
11 |
12 |
13 | // dm
14 | public static final Exception MemTooSmallException = new RuntimeException("Memory too small!");
15 | public static final Exception BadLogFileException = new RuntimeException("Bad log file!");
16 | public static Exception DataTooLargeException = new RuntimeException("Data too large!");
17 | public static Exception DataBaseBusyException = new RuntimeException("Database is too busy!");
18 |
19 | // tm
20 | public static final Exception BadXIDFileException = new RuntimeException("Bad XID file!");
21 |
22 | // vm
23 | public static Exception NullEntryException = new RuntimeException("Entry is null!");
24 | public static Exception DeadLockException = new RuntimeException("Deadlock detected!");
25 |
26 | // parser
27 | public static Exception InvalidCommandException = new RuntimeException("Command is invalid!");
28 | public static Exception TableNoIndexException = new RuntimeException("Table has no index!");
29 |
30 | // tbm
31 | public static Exception InvalidFieldException = new RuntimeException("Field is invalid!");
32 | public static Exception DuplicatedTableException = new RuntimeException("Table already exists!");
33 | public static Exception TableNotFoundException = new RuntimeException("Table does not exists!");
34 | public static Exception FieldNotIndexedException = new RuntimeException("Field is not indexed!");
35 | public static Exception InvalidLogOpException = new RuntimeException("Invalid logic operation!");
36 | public static Exception FieldNotFoundException = new RuntimeException("Field does not exists!");
37 | public static Exception ConcurrentUpdateException = new RuntimeException("Concurrent update happens!");
38 |
39 | // c/s
40 | public static Exception InvalidPkgDataException = new RuntimeException("Package data is invalid!");
41 | public static Exception NestedTransactionException = new RuntimeException("Nested transaction not supported!");
42 | public static Exception NoTransactionException = new RuntimeException("Not in transaction!");
43 | public static Exception InvalidMemException = new RuntimeException("Memory is invalid!");
44 | }
45 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/transport/Encoder.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.transport;
2 |
3 | import com.google.common.primitives.Bytes;
4 | import com.northeastern.edu.simpledb.common.Error;
5 |
6 | import java.util.Arrays;
7 |
8 | public class Encoder {
9 |
10 | public byte[] encode(Package pkg) {
11 | if (pkg.getErr() != null) {
12 | Exception err = pkg.getErr();
13 | String msg = "Internal Server Error!";
14 | if (err.getMessage() != null) {
15 | msg = err.getMessage();
16 | }
17 | return Bytes.concat(new byte[]{1}, msg.getBytes());
18 | } else return Bytes.concat(new byte[]{0}, pkg.getData());
19 | }
20 |
21 | public Package decode(byte[] data) throws Exception {
22 | if (data.length < 1) throw Error.InvalidPkgDataException;
23 | else if (data[0] == 0) return new Package(Arrays.copyOfRange(data, 1, data.length), null);
24 | else if (data[0] == 1) return new Package(null, new RuntimeException(new String(Arrays.copyOfRange(data, 1, data.length))));
25 | else throw Error.InvalidPkgDataException;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/transport/Package.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.transport;
2 |
3 | public class Package {
4 |
5 | private byte[] data;
6 | private Exception err;
7 |
8 | public Package(byte[] data, Exception err) {
9 | this.data = data;
10 | this.err = err;
11 | }
12 |
13 | public byte[] getData() {
14 | return data;
15 | }
16 |
17 | public Exception getErr() {
18 | return err;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/transport/Packager.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.transport;
2 |
3 | import org.apache.commons.codec.DecoderException;
4 |
5 | import java.io.IOException;
6 |
7 | public class Packager {
8 | private Transporter transporter;
9 | private Encoder encoder;
10 |
11 | public Packager(Transporter transporter, Encoder encoder) {
12 | this.transporter = transporter;
13 | this.encoder = encoder;
14 | }
15 |
16 | public void send(Package pkg) throws IOException {
17 | byte[] data = encoder.encode(pkg);
18 | transporter.send(data);
19 | }
20 |
21 | public Package receive() throws Exception {
22 | byte[] data = transporter.receive();
23 | return encoder.decode(data);
24 | }
25 |
26 | public void close() throws IOException {
27 | transporter.close();
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/com/northeastern/edu/simpledb/transport/Transporter.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.transport;
2 |
3 | import org.apache.commons.codec.DecoderException;
4 | import org.apache.commons.codec.binary.Hex;
5 |
6 | import java.io.*;
7 | import java.net.Socket;
8 |
9 | public class Transporter {
10 |
11 | private Socket socket;
12 | private BufferedReader reader;
13 | private BufferedWriter writer;
14 |
15 | public Transporter(Socket socket) throws IOException {
16 | this.socket = socket;
17 | this.reader = new BufferedReader(new InputStreamReader(socket.getInputStream()));
18 | this.writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream()));
19 | }
20 |
21 | public byte[] receive() throws IOException, DecoderException {
22 | String line = reader.readLine();
23 | if (line == null) close();
24 | return hexDecode(line);
25 | }
26 |
27 | public void send(byte[] data) throws IOException {
28 | String raw = hexEncode(data);
29 | writer.write(raw);
30 | writer.flush();
31 | }
32 |
33 | private String hexEncode(byte[] data) {
34 | return Hex.encodeHexString(data, true) + "\n";
35 | }
36 |
37 | private byte[] hexDecode(String line) throws DecoderException {
38 | return Hex.decodeHex(line);
39 | }
40 |
41 | public void close() throws IOException {
42 | this.writer.close();
43 | this.reader.close();
44 | this.socket.close();
45 | }
46 |
47 |
48 | }
49 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/common/AbstractCacheTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.common;
2 |
3 | import com.northeastern.edu.simpledb.common.Error;
4 |
5 | import java.util.Objects;
6 | import java.util.concurrent.*;
7 |
8 | import org.junit.jupiter.api.BeforeAll;
9 | import org.junit.jupiter.api.Test;
10 |
11 | import java.io.*;
12 |
13 | import static org.junit.jupiter.api.Assertions.*;
14 |
15 | public class AbstractCacheTest {
16 |
17 | private static final String SERIALIZE_SUFFIX = ".ser";
18 |
19 | private static final Integer THREAD_COUNT = 8;
20 |
21 | @BeforeAll
22 | static void prepareDataForTest() {
23 |
24 | for (int i = 1; i <= 2; i++) {
25 | // create an instance of the Data class
26 | Data data = new Data();
27 | data.setContent(String.valueOf(i));
28 |
29 | // specify the file path where you want to save the serialized object
30 | String filePath = data.getContent() + SERIALIZE_SUFFIX;
31 |
32 | // serialize the object and write it to the file
33 | try (FileOutputStream fileOut = new FileOutputStream(filePath);
34 | ObjectOutputStream objectOut = new ObjectOutputStream(fileOut)) {
35 |
36 | // write the object to the output stream
37 | objectOut.writeObject(data);
38 |
39 | System.out.println("Object has been serialized and written to " + filePath);
40 |
41 | } catch (IOException e) {
42 | e.printStackTrace();
43 | }
44 | }
45 | }
46 |
47 | @Test
48 | void testCacheIsFull_throwCacheFullException() throws Exception {
49 |
50 | int maxResource = 1;
51 | MyAbstractCache cache = new MyAbstractCache(maxResource);
52 |
53 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT);
54 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT);
55 |
56 | Future>[] futures = new Future[THREAD_COUNT];
57 |
58 | assertThrows(Error.CacheFullException.getClass(), () -> {
59 | for (int i = 0; i < THREAD_COUNT; i++) {
60 | final int finalI = i;
61 | futures[i] = executorService.submit(() -> {
62 | countDownLatch.countDown();
63 | try {
64 | countDownLatch.await();
65 | Data data = cache.get(finalI % 2 == 0 ? 1L : 2L);
66 | if (data != null) System.out.println("data content = " + data.getContent());
67 | } catch (Exception e) {
68 | throw new RuntimeException(e);
69 | }
70 | });
71 | }
72 |
73 | for (int i = 0; i < THREAD_COUNT; i++) {
74 | try {
75 | futures[i].get();
76 | } catch (InterruptedException | ExecutionException e) {
77 | throw new RuntimeException(e);
78 | }
79 | }
80 | });
81 |
82 | executorService.shutdown();
83 | executorService.awaitTermination(2, TimeUnit.SECONDS);
84 | }
85 |
86 | @Test
87 | void testMultiThreadGet_expectedNumberOfOutputEqualToThreadCount() throws Exception {
88 | int maxResource = 2;
89 | MyAbstractCache cache = new MyAbstractCache(maxResource);
90 |
91 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT);
92 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT);
93 |
94 | for (int i = 0; i < THREAD_COUNT; i++) {
95 | final int finalI = i;
96 | executorService.submit(() -> {
97 | countDownLatch.countDown();
98 | try {
99 | countDownLatch.await();
100 | Data data = cache.get(finalI % 2 == 0 ? 1L : 2L);
101 | if(data != null) System.out.println("data content = " + data.getContent());
102 | } catch (Exception e) {
103 | assertEquals(Error.CacheFullException, e);
104 | }
105 | });
106 | }
107 |
108 | executorService.shutdown();
109 | executorService.awaitTermination(2, TimeUnit.SECONDS);
110 | }
111 |
112 | @Test
113 | void testCacheRelease_expectedOnlyOneSerialization() throws InterruptedException {
114 | int maxResource = 2;
115 | MyAbstractCache cache = new MyAbstractCache(maxResource);
116 |
117 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT);
118 | CountDownLatch countDownLatch = new CountDownLatch(THREAD_COUNT);
119 |
120 | // submit multiple threads trying to access the same key concurrently
121 | for (int i = 0; i < THREAD_COUNT; i++) {
122 | final int finalI = i;
123 | executorService.submit(() -> {
124 | countDownLatch.countDown();
125 | try {
126 | countDownLatch.await();
127 | Data data = cache.get(1L);
128 | if (Thread.currentThread().getId() % 2 == 0) TimeUnit.SECONDS.sleep(2);
129 | System.out.println("Thread " + Thread.currentThread().getId() + " got data: " + data.getContent());
130 | // update data
131 | data.setContent(String.valueOf(finalI));
132 | // release the cache after processing
133 | cache.release(1L);
134 | } catch (Exception e) {
135 | throw new RuntimeException(e);
136 | }
137 | });
138 | }
139 |
140 | executorService.shutdown();
141 | executorService.awaitTermination(2, TimeUnit.SECONDS);
142 | }
143 | }
144 |
145 | class MyAbstractCache extends AbstractCache {
146 |
147 | private static final String SERIALIZE_SUFFIX = ".ser";
148 |
149 | public MyAbstractCache(int maxResource) {
150 | super(maxResource);
151 | }
152 |
153 | @Override
154 | protected Data getForCache(long key) {
155 | Data data = null;
156 |
157 | // specify the file path where the serialized object is stored
158 | String filePath = key + SERIALIZE_SUFFIX;
159 |
160 | // deserialize the object from the file
161 | try (FileInputStream fileIn = new FileInputStream(filePath);
162 | ObjectInputStream objectIn = new ObjectInputStream(fileIn)) {
163 |
164 | // read the object from the input stream
165 | data = (Data) objectIn.readObject();
166 |
167 | // access the content of the deserialized object
168 | System.out.println("Deserialized content: " + data.getContent());
169 |
170 | } catch (IOException | ClassNotFoundException e) {
171 | e.printStackTrace();
172 | }
173 | return data;
174 | }
175 |
176 | @Override
177 | protected void releaseForCache(Data obj) {
178 |
179 | // specify the file path where you want to save the serialized object
180 | String filePath = obj.getContent() + SERIALIZE_SUFFIX;
181 |
182 | // serialize the object and write it to the file
183 | try (FileOutputStream fileOut = new FileOutputStream(filePath);
184 | ObjectOutputStream objectOut = new ObjectOutputStream(fileOut)) {
185 |
186 | // write the object to the output stream
187 | objectOut.writeObject(obj);
188 |
189 | System.out.println("Object has been serialized and written to " + filePath);
190 |
191 | } catch (IOException e) {
192 | e.printStackTrace();
193 | }
194 | }
195 |
196 | }
197 |
198 | class Data implements Serializable {
199 |
200 | public static final long SERIALIZE_ID = 123456789L;
201 |
202 | private String content;
203 |
204 | public String getContent() {
205 | return content;
206 | }
207 |
208 | public void setContent(String content) {
209 | this.content = content;
210 | }
211 |
212 | @Override
213 | public boolean equals(Object o) {
214 | if (this == o) return true;
215 | if (o == null || getClass() != o.getClass()) return false;
216 | Data data = (Data) o;
217 | return Objects.equals(content, data.content);
218 | }
219 |
220 | @Override
221 | public int hashCode() {
222 | return Objects.hash(content);
223 | }
224 | }
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/dm/DataManagerTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.SubArray;
4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
7 | import com.northeastern.edu.simpledb.backend.utils.Panic;
8 | import com.northeastern.edu.simpledb.backend.utils.RandomUtil;
9 | import org.checkerframework.checker.units.qual.C;
10 | import org.junit.jupiter.api.AfterAll;
11 | import org.junit.jupiter.api.Assertions;
12 | import org.junit.jupiter.api.BeforeAll;
13 | import org.junit.jupiter.api.Test;
14 |
15 | import java.io.File;
16 | import java.nio.charset.StandardCharsets;
17 | import java.security.SecureRandom;
18 | import java.util.*;
19 | import java.util.concurrent.ConcurrentHashMap;
20 | import java.util.concurrent.CountDownLatch;
21 | import java.util.concurrent.locks.Lock;
22 | import java.util.concurrent.locks.ReentrantLock;
23 |
24 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX;
25 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX;
26 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX;
27 |
28 | public class DataManagerTest {
29 |
30 | private static final int workerNum = 10;
31 | private static final int tasksNum = 50;
32 |
33 | private static final int totalData = 10;
34 | static CountDownLatch cdl;
35 | static TransactionManager tm;
36 |
37 | static DataManger dm;
38 |
39 | static Lock lock;
40 |
41 | static List uids;
42 |
43 | static Random random = new SecureRandom();
44 | static ConcurrentHashMap map;
45 |
46 | @BeforeAll
47 | static void setup() {
48 | uids = new ArrayList<>();
49 | tm = TransactionManager.create("dm-test");
50 | dm = DataMangerHandler.create("dm-test", PageCache.PAGE_SIZE * 10, tm);
51 | cdl = new CountDownLatch(workerNum);
52 | lock = new ReentrantLock();
53 | map = new ConcurrentHashMap<>();
54 |
55 | for (int i = 0; i < tasksNum; i++) {
56 | String s = UUID.randomUUID().toString();
57 | map.put(i, s);
58 | }
59 | }
60 |
61 | @AfterAll
62 | static void cleanTestEnv() {
63 | new File("dm-test" + LOG_SUFFIX).delete();
64 | new File("dm-test" + DB_SUFFIX).delete();
65 | new File("dm-test" + XID_SUFFIX).delete();
66 | }
67 |
68 | @Test
69 | void testMultiThread_expectedMapContainsAllElements() {
70 | for (int i = 0; i < workerNum; i++) {
71 | new Thread(() -> worker()).start();
72 | }
73 | try {
74 | cdl.await();
75 | } catch (InterruptedException e) {
76 | throw new RuntimeException(e);
77 | }
78 | }
79 |
80 |
81 | private static final int dataRotation = 50;
82 |
83 | private void worker() {
84 | try {
85 | for (int i = 0; i < tasksNum; i++) {
86 | int v;
87 | if ((v = random.nextInt() % 100) > dataRotation) {
88 | String s = map.get(random.nextInt(tasksNum));
89 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
90 | long uid = dm.insert(0, bytes);
91 | try {
92 | lock.lock();
93 | uids.add(uid);
94 | } finally {
95 | lock.unlock();
96 | }
97 | } else {
98 | try {
99 | lock.lock();
100 | if (uids.size() == 0) continue;
101 | } finally {
102 | lock.unlock();
103 | }
104 | int tmp = Math.abs(random.nextInt()) % uids.size();
105 | Long uid = uids.get(tmp);
106 | DataItem dataItem = dm.read(uid);
107 |
108 | dataItem.rLock();
109 | SubArray sa = dataItem.data();
110 | byte[] bytes = Arrays.copyOfRange(sa.raw, sa.start, sa.end);
111 | Assertions.assertTrue(map.contains(new String(bytes)));
112 | dataItem.rUnLock();
113 | }
114 | }
115 | } catch (Exception e){
116 | Panic.panic(e);
117 | } finally {
118 | cdl.countDown();
119 | }
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/dm/RecoverTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm;
2 |
3 | import com.northeastern.edu.simpledb.backend.common.SubArray;
4 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
5 | import com.northeastern.edu.simpledb.backend.dm.dataItem.DataItem;
6 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger;
7 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
8 | import org.junit.jupiter.api.AfterAll;
9 | import org.junit.jupiter.api.Assertions;
10 | import org.junit.jupiter.api.BeforeAll;
11 | import org.junit.jupiter.api.Test;
12 |
13 | import java.io.File;
14 | import java.lang.reflect.Field;
15 | import java.nio.charset.StandardCharsets;
16 | import java.util.Arrays;
17 |
18 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX;
19 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX;
20 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX;
21 |
22 | public class RecoverTest {
23 |
24 | private static DataManger dm;
25 | private static TransactionManager tm;
26 |
27 | private static long xid;
28 |
29 | @BeforeAll
30 | static void setup() {
31 | tm = TransactionManager.create("recover-test");
32 | dm = DataMangerHandler.create("recover-test", PageCache.PAGE_SIZE * 10, tm);
33 | xid = tm.begin();
34 | }
35 |
36 | @AfterAll
37 | static void cleanTestEnv() {
38 | new File("recover-test" + LOG_SUFFIX).delete();
39 | new File("recover-test" + DB_SUFFIX).delete();
40 | new File("recover-test" + XID_SUFFIX).delete();
41 | }
42 |
43 | @Test
44 | void testRecover_expectedRedoWorksAndDataItemWithCorrespondingUidBecomesInvalid() {
45 | long uid;
46 | try {
47 | uid = dm.insert(xid, "hello db!".getBytes(StandardCharsets.UTF_8));
48 | DataItem dataItem0 = dm.read(uid);
49 | dataItem0.rLock();
50 | SubArray data0 = dataItem0.data();
51 | dataItem0.rUnLock();
52 | byte[] bytes = Arrays.copyOfRange(data0.raw, data0.start, data0.end);
53 | String actualStr = new String(bytes);
54 | Assertions.assertEquals("hello db!", actualStr);
55 |
56 | Field loggerFiled = dm.getClass().getDeclaredField("logger");
57 | loggerFiled.setAccessible(true);
58 | Logger reloadedLog = Logger.open("recover-test");
59 | loggerFiled.set(dm, reloadedLog);
60 |
61 | Field pageCacheField = dm.getClass().getDeclaredField("pageCache");
62 | pageCacheField.setAccessible(true);
63 | PageCache pageCache = (PageCache) pageCacheField.get(dm);
64 |
65 | Recover.recover(tm, reloadedLog, pageCache);
66 |
67 | Assertions.assertNull(dm.read(uid));
68 |
69 | } catch (Exception e) {
70 | throw new RuntimeException(e);
71 | }
72 | }
73 | }
74 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/dm/logger/LoggerTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.dm.logger;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.logger.Logger;
4 | import org.junit.jupiter.api.*;
5 |
6 | import java.io.IOException;
7 | import java.lang.reflect.Method;
8 | import java.nio.charset.StandardCharsets;
9 | import java.nio.file.Files;
10 | import java.nio.file.Path;
11 |
12 | @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
13 | public class LoggerTest {
14 |
15 | private static final short OF_CHECKSUM = 4;
16 | private static Logger logger;
17 |
18 | private static final String LOG_NAME = "bin";
19 |
20 | @BeforeAll
21 | static void setup() {
22 | logger = Logger.create(LOG_NAME);
23 | }
24 |
25 | @AfterAll
26 | static void cleanTestEnv() {
27 | try {
28 | Files.deleteIfExists(Path.of(LOG_NAME + Logger.LOG_SUFFIX));
29 | } catch (IOException e) {
30 | throw new RuntimeException(e);
31 | } finally {
32 | logger.close();
33 | }
34 | }
35 |
36 | @Test
37 | @Order(1)
38 | void testInit_expectedFCPositionEqualsToOffsetCheckSumAfterInit() {
39 | try {
40 | Method initMethod = logger.getClass().getDeclaredMethod("init");
41 | initMethod.setAccessible(true);
42 | initMethod.invoke(logger);
43 | Assertions.assertEquals(OF_CHECKSUM, logger.getFc().position());
44 | } catch (Exception e) {
45 | throw new RuntimeException(e);
46 | }
47 | }
48 |
49 | @Test
50 | @Order(2)
51 | void testLogOnce_expectedFCPositionEqualsToOffsetCheckSumAfterLog() {
52 | String sql = "select * from test";
53 | byte[] bytes = sql.getBytes(StandardCharsets.UTF_8);
54 | logger.log(bytes);
55 | try {
56 | Assertions.assertEquals(OF_CHECKSUM, logger.getFc().position());
57 | } catch (IOException e) {
58 | throw new RuntimeException(e);
59 | }
60 | }
61 |
62 | @Test
63 | @Order(3)
64 | void testLogOnce_expectedSameSQL() {
65 | String expected = "select * from test";
66 | logger = Logger.open(LOG_NAME);
67 | byte[] log = logger.next();
68 | assert log != null;
69 | String actual = new String(log);
70 | Assertions.assertEquals(expected, actual);
71 | }
72 |
73 | @Test
74 | @Order(4)
75 | void testLogMultipleTimes_expectedSameSQL() {
76 | String sql1 = "select * from test";
77 | logger.log(sql1.getBytes(StandardCharsets.UTF_8));
78 |
79 | String sql2 = "select * from test where a = 1";
80 | logger.log(sql2.getBytes(StandardCharsets.UTF_8));
81 |
82 | String sql3 = "update from test set b = 3 where a = 1";
83 | logger.log(sql3.getBytes(StandardCharsets.UTF_8));
84 |
85 | logger = Logger.open(LOG_NAME);
86 |
87 | logger.next();
88 |
89 | byte[] log1 = logger.next();
90 | assert log1 != null;
91 | String res1 = new String(log1);
92 | Assertions.assertEquals(sql1, res1);
93 |
94 | byte[] log2 = logger.next();
95 | assert log2 != null;
96 | String res2 = new String(log2);
97 | Assertions.assertEquals(sql2, res2);
98 |
99 | byte[] log3 = logger.next();
100 | assert log3 != null;
101 | String res3 = new String(log3);
102 | Assertions.assertEquals(sql3, res3);
103 | }
104 |
105 | }
106 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/im/BPlusTreeTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.im;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler;
5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
7 | import org.junit.jupiter.api.AfterAll;
8 | import org.junit.jupiter.api.Test;
9 |
10 | import java.io.File;
11 | import java.util.List;
12 |
13 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX;
14 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX;
15 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX;
16 |
17 | public class BPlusTreeTest {
18 |
19 | private static final String TEST_NAME = "bptree-test";
20 |
21 | @AfterAll
22 | static void cleanTestEnv() {
23 | new File(TEST_NAME + LOG_SUFFIX).delete();
24 | new File(TEST_NAME + DB_SUFFIX).delete();
25 | new File(TEST_NAME + XID_SUFFIX).delete();
26 | }
27 |
28 | @Test
29 | void testBuildTree_expectedNoException() throws Exception {
30 | DataManger dm = DataMangerHandler.create(TEST_NAME, PageCache.PAGE_SIZE * 10, TransactionManager.create(TEST_NAME));
31 |
32 | long root = BPlusTree.create(dm);
33 | BPlusTree tree = BPlusTree.load(root, dm);
34 |
35 | int lim = 15;
36 | for (int i = 1; i <= lim; i++) {
37 | tree.insert(i, i);
38 | }
39 | for (int i = 1; i <= lim; i++) {
40 | List uids = tree.search(i);
41 | // System.out.println("=============================");
42 | assert uids.size() == 1;
43 | assert uids.get(0) == i;
44 | }
45 |
46 | /*
47 | tree.insert(16, 16);
48 | tree.insert(17, 17);
49 | tree.insert(18, 18);
50 | List uids = tree.search(6);
51 | */
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/pageIndex/PageIndexTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.pageIndex;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
4 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageIndex;
5 | import com.northeastern.edu.simpledb.backend.dm.pageIndex.PageInfo;
6 | import org.junit.jupiter.api.Test;
7 |
8 | public class PageIndexTest {
9 |
10 | @Test
11 | void testPageIndex_expectedNoException() {
12 | PageIndex pageIndex = new PageIndex();
13 | int threshold = PageCache.PAGE_SIZE / 40;
14 | for (int i = 0; i < 40; i++) {
15 | pageIndex.add(i, i * threshold);
16 | pageIndex.add(i, i * threshold);
17 | pageIndex.add(i, i * threshold);
18 | }
19 |
20 | for (int i = 0; i < 39; i++) {
21 | PageInfo pageInfo = pageIndex.select(i * threshold);
22 | assert pageInfo != null;
23 | assert pageInfo.pageNumber == i + 1;
24 | }
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/parser/ParserTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser;
2 |
3 | import com.google.gson.Gson;
4 | import com.northeastern.edu.simpledb.backend.parser.statement.*;
5 | import org.junit.jupiter.api.Test;
6 |
7 | import java.util.Arrays;
8 |
9 | class ParserTest {
10 |
11 | @Test
12 | void testCreate() throws Exception {
13 | String stat = "create table student id int32, name string, uid int64, (index name id uid)";
14 | Object res = Parser.Parse(stat.getBytes());
15 | Create create = (Create) res;
16 | assert "student".equals(create.tableName);
17 | System.out.println("Create");
18 | for (int i = 0; i < create.fieldName.length; i++) {
19 | System.out.println(create.fieldName[i] + ":" + create.fieldType[i]);
20 | }
21 | System.out.println(Arrays.toString(create.index));
22 | System.out.println("======================");
23 | }
24 |
25 | @Test
26 | void testBegin() throws Exception {
27 | String stat = "begin isolation level read committed";
28 | Object res = Parser.Parse(stat.getBytes());
29 | Begin begin = (Begin) res;
30 | assert !begin.isRepeatableRead;
31 |
32 | stat = "begin";
33 | res = Parser.Parse(stat.getBytes());
34 | begin = (Begin) res;
35 | assert !begin.isRepeatableRead;
36 |
37 | stat = "begin isolation level repeatable read";
38 | res = Parser.Parse(stat.getBytes());
39 | begin = (Begin) res;
40 | assert begin.isRepeatableRead;
41 | }
42 |
43 | @Test
44 | void testRead() throws Exception {
45 | String stat = "select name, id, age from student where id > 1 and id < 4";
46 | Object res = Parser.Parse(stat.getBytes());
47 | Select select = (Select) res;
48 | assert "student".equals(select.tableName);
49 | Gson gson = new Gson();
50 | System.out.println("Select");
51 | System.out.println(gson.toJson(select.fields));
52 | System.out.println(gson.toJson(select.where));
53 | System.out.println("======================");
54 | }
55 |
56 | @Test
57 | void testInsert() throws Exception {
58 | String stat = "insert into student values 5 \"jack\" 22";
59 | Object res = Parser.Parse(stat.getBytes());
60 | Insert insert = (Insert) res;
61 | Gson gson = new Gson();
62 | System.out.println("Insert");
63 | System.out.println(gson.toJson(insert));
64 | System.out.println("======================");
65 | }
66 |
67 | @Test
68 | void testDelete() throws Exception {
69 | String stat = "delete from student where name = \"jack\"";
70 | Object res = Parser.Parse(stat.getBytes());
71 | Delete delete = (Delete) res;
72 | Gson gson = new Gson();
73 | System.out.println("Delete");
74 | System.out.println(gson.toJson(delete));
75 | System.out.println("======================");
76 | }
77 |
78 | @Test
79 | void testShow() throws Exception {
80 | String stat = "show";
81 | Object res = Parser.Parse(stat.getBytes());
82 | Show show = (Show) res;
83 | Gson gson = new Gson();
84 | System.out.println("Show");
85 | System.out.println(gson.toJson(show));
86 | System.out.println("======================");
87 | }
88 |
89 | @Test
90 | void testUpdate() throws Exception {
91 | String stat = "update student set name = \"jack\" where id = 5";
92 | Object res = Parser.Parse(stat.getBytes());
93 | Update update = (Update) res;
94 | Gson gson = new Gson();
95 | System.out.println("Update");
96 | System.out.println(gson.toJson(update));
97 | System.out.println("======================");
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/parser/TokenizerTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.parser;
2 |
3 | import org.junit.jupiter.api.Assertions;
4 | import org.junit.jupiter.api.BeforeEach;
5 | import org.junit.jupiter.api.Test;
6 | import org.mockito.InjectMocks;
7 | import org.mockito.Mock;
8 | import org.mockito.MockitoAnnotations;
9 |
10 | import java.nio.charset.StandardCharsets;
11 | import java.util.Objects;
12 |
13 | import static org.junit.jupiter.api.Assertions.*;
14 | import static org.mockito.Mockito.*;
15 |
16 | class TokenizerTest {
17 |
18 | @Mock
19 | Exception err;
20 |
21 | @InjectMocks
22 | Tokenizer tokenizer;
23 |
24 | @BeforeEach
25 | void setUp() {
26 | MockitoAnnotations.openMocks(this);
27 | }
28 |
29 | @Test
30 | void testValidInput_expectedNoException() {
31 | String s = "SELECT * FROM employee WHERE name = 'jack'";
32 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
33 | Tokenizer tokenizer = new Tokenizer(bytes);
34 |
35 | assertDoesNotThrow(() -> {
36 | assertEquals("SELECT", tokenizer.peek());
37 | tokenizer.pop();
38 |
39 | assertEquals("*", tokenizer.peek());
40 | tokenizer.pop();
41 | });
42 | }
43 |
44 | @Test
45 | void testInValidInput_expectedRuntimeException() {
46 | String s = " ^ ";
47 | byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
48 | Tokenizer tokenizer = new Tokenizer(bytes);
49 |
50 | assertThrows(RuntimeException.class, () -> {
51 | String token;
52 | while (!"".equals(token = tokenizer.peek())) {
53 | System.out.println("token = " + token);
54 | tokenizer.pop();
55 | }
56 | });
57 | }
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/tm/TransactionManagerTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.tm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.page.Page;
4 | import org.checkerframework.checker.units.qual.C;
5 | import org.junit.jupiter.api.AfterAll;
6 | import org.junit.jupiter.api.BeforeAll;
7 | import org.junit.jupiter.api.Test;
8 |
9 | import java.io.File;
10 | import java.nio.file.Files;
11 | import java.security.SecureRandom;
12 | import java.util.Map;
13 | import java.util.Random;
14 | import java.util.concurrent.ConcurrentHashMap;
15 | import java.util.concurrent.CountDownLatch;
16 | import java.util.concurrent.locks.Lock;
17 | import java.util.concurrent.locks.ReentrantLock;
18 |
19 | public class TransactionManagerTest {
20 |
21 | static Random random = new SecureRandom();
22 |
23 | private int transCnt = 0;
24 | private static final int noWorks = 300;
25 | private static final int noWorkers = 50;
26 | private static TransactionManager tm;
27 | private static Map transMap;
28 | private static Lock lock;
29 | private static CountDownLatch cdl;
30 |
31 |
32 | @BeforeAll
33 | static void setup() {
34 | tm = TransactionManager.create("tm-test");
35 | lock = new ReentrantLock();
36 | cdl = new CountDownLatch(noWorkers);
37 | transMap = new ConcurrentHashMap<>();
38 | }
39 |
40 | @AfterAll
41 | static void cleanTestEnv() {
42 | assert new File("tm-test" + TransactionManager.XID_SUFFIX).delete();
43 | }
44 |
45 | /**
46 | * let's suppose ConcurrentHashMap is thread-safe, randomly
47 | * starting a transaction, and put xid, status into ConcurrentHashMap
48 | * at same time, compare the state in ConcurrentHashMap with
49 | * state provided by Transaction Manager randomly.
50 | */
51 | @Test
52 | void testMultiThread_expectedNoException() {
53 | for (int i = 0; i < noWorkers; i++) {
54 | Runnable r = () -> worker();
55 | new Thread(r).run();
56 | }
57 | try {
58 | cdl.await();
59 | } catch (InterruptedException e) {
60 | e.printStackTrace();
61 | }
62 | }
63 |
64 | private void worker() {
65 | boolean inTans = false;
66 | long transXID = 0;
67 | for (int i = 0; i < noWorks; i++) {
68 | int op = Math.abs(random.nextInt(6));
69 | if (op == 0) { // do transaction
70 | lock.lock();
71 | if (!inTans) { // starting a transaction synchronously in TM and ConcurrentHashMap
72 | long xid = tm.begin();
73 | transMap.put(xid, (byte) 0);
74 | transCnt++;
75 | transXID = xid;
76 | inTans = true;
77 | } else { // determine if commit or abort transaction randomly
78 | int state = (random.nextInt(Integer.MAX_VALUE) % 2) + 1;
79 | switch (state) {
80 | case 1:
81 | tm.commit(transXID);
82 | break;
83 | case 2:
84 | tm.abort(transXID);
85 | break;
86 | }
87 | transMap.put(transXID, (byte) state);
88 | inTans = false;
89 | }
90 | lock.unlock();
91 | } else { // check state
92 | lock.lock();
93 | if (transCnt > 0) {
94 | long xid = (long) ((random.nextInt(Integer.MAX_VALUE) % transCnt) + 1);
95 | byte state = transMap.get(xid);
96 | boolean ok = false;
97 | switch (state) {
98 | case 0:
99 | ok = tm.isActive(xid);
100 | break;
101 | case 1:
102 | ok = tm.isCommitted(xid);
103 | break;
104 | case 2:
105 | ok = tm.isAborted(xid);
106 | break;
107 | }
108 | assert ok;
109 | }
110 | lock.unlock();
111 | }
112 | }
113 | cdl.countDown();
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/vm/LockTableTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import org.junit.jupiter.api.Assertions;
5 | import org.junit.jupiter.api.BeforeEach;
6 | import org.junit.jupiter.api.Test;
7 | import org.mockito.InjectMocks;
8 | import org.mockito.Mock;
9 | import org.mockito.MockitoAnnotations;
10 | import org.mockito.Spy;
11 |
12 | import java.util.ArrayList;
13 | import java.util.HashMap;
14 | import java.util.List;
15 | import java.util.Map;
16 | import java.util.concurrent.locks.Lock;
17 |
18 | import static org.junit.jupiter.api.Assertions.assertThrows;
19 |
20 | class LockTableTest {
21 | @Spy
22 | Map> x2u = new HashMap<>();
23 | @InjectMocks
24 | LockTable lockTable;
25 |
26 | @BeforeEach
27 | void setUp() {
28 | MockitoAnnotations.openMocks(this);
29 | }
30 |
31 | /*
32 | eg:
33 | dfs(xid2)
34 | xid2 try to acquire uid1, but uid1 is holding by xid1
35 | dfs(xid1)
36 | xid1 is waiting for uid2, but uid2 is holding by xid2
37 | */
38 | @Test
39 | void testSingleAdd_expectedRuntimeException() throws Exception {
40 | try {
41 | lockTable.add(1, 1);
42 | lockTable.add(2, 2);
43 | lockTable.add(1, 2);
44 | } catch (Exception e) {
45 | Panic.panic(e);
46 | }
47 |
48 | assertThrows(RuntimeException.class, () -> lockTable.add(2, 1));
49 | }
50 |
51 | @Test
52 | void testRemove_expectedNoException() {
53 | try {
54 | lockTable.add(1, 1);
55 | lockTable.add(1, 2);
56 | lockTable.add(1, 3);
57 | lockTable.add(1, 4);
58 | lockTable.add(2, 4);
59 | lockTable.remove(1);
60 | } catch (Exception e) {
61 | Panic.panic(e);
62 | }
63 | }
64 |
65 | @Test
66 | void testMultiAdd_expectedRuntimeException() {
67 | for (long i = 1; i <= 100; i ++) {
68 | try {
69 | Lock o = lockTable.add(i, i);
70 | if(o != null) {
71 | Runnable r = () -> {
72 | o.lock();
73 | o.unlock();
74 | };
75 | new Thread(r).start();
76 | }
77 | } catch (Exception e) {
78 | Panic.panic(e);
79 | }
80 | }
81 |
82 | for (long i = 1; i <= 99; i ++) {
83 | try {
84 | Lock o = lockTable.add(i, i+1);
85 | if(o != null) {
86 | Runnable r = () -> {
87 | o.lock();
88 | o.unlock();
89 | };
90 | new Thread(r).start();
91 | }
92 | } catch (Exception e) {
93 | Panic.panic(e);
94 | }
95 | }
96 |
97 | assertThrows(RuntimeException.class, () -> lockTable.add(100, 1));
98 | lockTable.remove(23);
99 |
100 | try {
101 | lockTable.add(100, 1);
102 | } catch (Exception e) {
103 | Panic.panic(e);
104 | }
105 | }
106 |
107 |
108 | @Test
109 | void testPutIntoList_expectedNoException() {
110 | x2u.computeIfAbsent(1L, k -> new ArrayList<>()).add(0, 1L);
111 | assert x2u.size() > 0 && x2u.get(1L) != null && x2u.get(1L).size() == 1;
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/backend/vm/VersionManagerTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.backend.vm;
2 |
3 | import com.northeastern.edu.simpledb.backend.dm.DataManger;
4 | import com.northeastern.edu.simpledb.backend.dm.DataMangerHandler;
5 | import com.northeastern.edu.simpledb.backend.dm.cache.PageCache;
6 | import com.northeastern.edu.simpledb.backend.tm.TransactionManager;
7 | import org.junit.jupiter.api.*;
8 | import org.mockito.InjectMocks;
9 | import org.mockito.MockitoAnnotations;
10 |
11 | import java.io.File;
12 | import java.nio.charset.StandardCharsets;
13 |
14 | import static com.northeastern.edu.simpledb.backend.dm.cache.PageCache.DB_SUFFIX;
15 | import static com.northeastern.edu.simpledb.backend.dm.logger.Logger.LOG_SUFFIX;
16 | import static com.northeastern.edu.simpledb.backend.tm.TransactionManager.XID_SUFFIX;
17 |
18 | class VersionManagerTest {
19 |
20 | private static final String TEST_NAME = "vm-test";
21 |
22 | static TransactionManager tm;
23 | static DataManger dm;
24 | static VersionManagerHandler vm;
25 |
26 | @BeforeEach
27 | void setUp() {
28 | MockitoAnnotations.openMocks(this);
29 | }
30 |
31 | @BeforeAll
32 | static void setUpEnv() {
33 | tm = TransactionManager.create(TEST_NAME);
34 | dm = DataMangerHandler.create(TEST_NAME, PageCache.PAGE_SIZE * 10, tm);
35 | vm = VersionManagerHandler.newVersionManager(tm, dm);
36 | }
37 |
38 | @AfterAll
39 | static void cleanTestEnv() {
40 | new File(TEST_NAME + LOG_SUFFIX).delete();
41 | new File(TEST_NAME + DB_SUFFIX).delete();
42 | new File(TEST_NAME + XID_SUFFIX).delete();
43 | }
44 |
45 | @Test
46 | void testRead_expectedNoException() throws Exception {
47 | long xid = vm.begin(1);
48 | long uid = vm.insert(xid, "version manager read test".getBytes(StandardCharsets.UTF_8));
49 | Assertions.assertEquals(new String(vm.read(xid, uid)), "version manager read test");
50 | vm.commit(xid);
51 | }
52 |
53 | @Test
54 | void testDelete_expectedNoException() throws Exception {
55 | long xid = vm.begin(1);
56 | long uid = vm.insert(xid, "version manager delete test".getBytes(StandardCharsets.UTF_8));
57 | Assertions.assertEquals(vm.delete(xid, uid), Boolean.TRUE);
58 | Assertions.assertNull(vm.read(xid, uid));
59 | vm.commit(xid);
60 | }
61 |
62 | @Test
63 | void testAbort_expectedRollBackSuccess() throws Exception {
64 | long xid = vm.begin(1);
65 | long uid = vm.insert(xid, "version manager abort test".getBytes(StandardCharsets.UTF_8));
66 | vm.abort(xid);
67 | Assertions.assertNull(vm.read(xid, uid));
68 | }
69 |
70 | }
--------------------------------------------------------------------------------
/src/test/java/com/northeastern/edu/simpledb/transport/PackageTest.java:
--------------------------------------------------------------------------------
1 | package com.northeastern.edu.simpledb.transport;
2 |
3 | import com.northeastern.edu.simpledb.backend.utils.Panic;
4 | import org.junit.jupiter.api.Test;
5 |
6 | import java.net.InetAddress;
7 | import java.net.ServerSocket;
8 | import java.net.Socket;
9 |
10 | public class PackageTest {
11 |
12 | @Test
13 | void testPackager() throws Exception {
14 | new Thread(() -> {
15 | try {
16 | ServerSocket welcomeSocket = new ServerSocket(10888);
17 | Socket socket = welcomeSocket.accept();
18 | Transporter t = new Transporter(socket);
19 | Encoder e = new Encoder();
20 | Packager p = new Packager(t, e);
21 |
22 | Package one = p.receive();
23 | assert "pkg1 test".equals(new String(one.getData()));
24 | Package two = p.receive();
25 | assert "pkg2 test".equals(new String(two.getData()));
26 |
27 | p.send(new Package("pkg3 test".getBytes(), null));
28 | welcomeSocket.close();
29 | } catch (Exception e) {
30 | Panic.panic(e);
31 | }
32 | }).start();
33 |
34 | Socket socket = new Socket(InetAddress.getLocalHost().getHostAddress(), 10888);
35 | Transporter t = new Transporter(socket);
36 | Encoder e = new Encoder();
37 | Packager p = new Packager(t, e);
38 |
39 | p.send(new Package("pkg1 test".getBytes(), null));
40 | p.send(new Package("pkg2 test".getBytes(), null));
41 |
42 | Package three = p.receive();
43 | assert "pkg3 test".equals(new String(three.getData()));
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/start-client.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -eu
4 |
5 | PACKAGE_NAME=com.northeastern.edu.simpledb.client.Launcher
6 |
7 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME"
--------------------------------------------------------------------------------
/start-server.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -eu
4 |
5 | PACKAGE_NAME=com.northeastern.edu.simpledb.backend.Launcher
6 |
7 | mvn clean compile
8 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME" -Dexec.args="-create simple-db"
9 | mvn exec:java -Dexec.mainClass="$PACKAGE_NAME" -Dexec.args="-open simple-db"
--------------------------------------------------------------------------------