├── .idea ├── .name ├── copyright │ └── profiles_settings.xml ├── scopes │ └── scope_settings.xml ├── encodings.xml ├── vcs.xml ├── modules.xml ├── gradle.xml ├── misc.xml ├── compiler.xml ├── uiDesigner.xml └── workspace.xml ├── .gradle └── 1.4 │ └── taskArtifacts │ ├── cache.properties.lock │ ├── cache.properties │ ├── fileHashes.bin │ ├── fileSnapshots.bin │ ├── taskArtifacts.bin │ └── outputFileStates.bin ├── .DS_Store ├── src ├── .DS_Store ├── main │ └── java │ │ └── com │ │ └── pwc │ │ ├── bitcask │ │ ├── Index.java │ │ ├── Indexer.java │ │ ├── test │ │ │ └── Student.java │ │ └── BitCask.java │ │ └── object │ │ └── Serialization.java └── test │ └── java │ └── com │ └── pwc │ └── bitcask │ └── BitCaskTest.java ├── pom.xml ├── BitCask.iml └── README.md /.idea/.name: -------------------------------------------------------------------------------- 1 | BitCask -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/cache.properties.lock: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/cache.properties: -------------------------------------------------------------------------------- 1 | #Sun Sep 29 12:56:52 CST 2013 2 | -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/src/.DS_Store -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/.gradle/1.4/taskArtifacts/fileHashes.bin -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/fileSnapshots.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/.gradle/1.4/taskArtifacts/fileSnapshots.bin -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/taskArtifacts.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/.gradle/1.4/taskArtifacts/taskArtifacts.bin -------------------------------------------------------------------------------- /.gradle/1.4/taskArtifacts/outputFileStates.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/darlinglele/bitcask/HEAD/.gradle/1.4/taskArtifacts/outputFileStates.bin -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /.idea/scopes/scope_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/main/java/com/pwc/bitcask/Index.java: -------------------------------------------------------------------------------- 1 | package com.pwc.bitcask; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Index implements Serializable { 6 | public final String key; 7 | public final long offset; 8 | public final int size; 9 | public final String fileName; 10 | 11 | public Index(String key, String name, long offset, int size) { 12 | this.key = key; 13 | this.fileName = name; 14 | this.offset = offset; 15 | this.size = size; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/pwc/bitcask/Indexer.java: -------------------------------------------------------------------------------- 1 | package com.pwc.bitcask; 2 | 3 | import java.io.Serializable; 4 | import java.util.HashMap; 5 | 6 | public class Indexer implements Serializable { 7 | private final HashMap map; 8 | 9 | public Indexer() { 10 | this.map = new HashMap<>(); 11 | } 12 | 13 | public Index get(String key) { 14 | return map.get(key); 15 | } 16 | 17 | public void put(String key, Index index) { 18 | map.put(key, index); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | BitCask 8 | BitCask 9 | 1.0-SNAPSHOT 10 | 11 | 12 | junit 13 | junit 14 | 4.4 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/java/com/pwc/object/Serialization.java: -------------------------------------------------------------------------------- 1 | package com.pwc.object; 2 | 3 | import java.io.*; 4 | 5 | public class Serialization { 6 | 7 | public static byte[] serialize(Object obj) throws IOException { 8 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 9 | ObjectOutputStream os = new ObjectOutputStream(out); 10 | os.writeObject(obj); 11 | return out.toByteArray(); 12 | } 13 | 14 | public static Object deserialize(byte[] data) throws IOException, ClassNotFoundException { 15 | ByteArrayInputStream in = new ByteArrayInputStream(data); 16 | ObjectInputStream is = new ObjectInputStream(in); 17 | return is.readObject(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/java/com/pwc/bitcask/BitCaskTest.java: -------------------------------------------------------------------------------- 1 | package com.pwc.bitcask; 2 | 3 | import com.pwc.bitcask.test.Student; 4 | import org.junit.Test; 5 | 6 | import static org.hamcrest.core.Is.is; 7 | import static org.junit.Assert.assertThat; 8 | 9 | public class BitCaskTest { 10 | @Test 11 | public void should_return_the_object_has_been_putted() throws Exception { 12 | BitCask studentBitCask = BitCask.of("Student"); 13 | int i = 0; 14 | while (i < 100000) { 15 | String key = String.valueOf(i); 16 | Student value = new Student("401612002", "name", "age", "hobbit"); 17 | studentBitCask.put(key, value); 18 | assertThat(studentBitCask.get(key), is(value)); 19 | i++; 20 | } 21 | studentBitCask.dumpIndexTo("Student.index"); 22 | } 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/pwc/bitcask/test/Student.java: -------------------------------------------------------------------------------- 1 | package com.pwc.bitcask.test; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Student implements Serializable { 6 | private final String id; 7 | private final String name; 8 | private final String age; 9 | private final String hobbit; 10 | private static final long serialVersionUID = 14334; 11 | 12 | public Student(String id, String name, String age, String hobbit) { 13 | this.id = id; 14 | this.name = name; 15 | this.age = age; 16 | this.hobbit = hobbit; 17 | } 18 | 19 | @Override 20 | public boolean equals(Object other) { 21 | if (this == other) { 22 | return true; 23 | } 24 | 25 | if (other == null || !(other instanceof Student)) { 26 | return false; 27 | } 28 | 29 | Student student = (Student) other; 30 | return student.id.equals(this.id) && student.name.equals(this.name); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /BitCask.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Bitcask简洁、优雅的key/value存储引擎 2 | ========== 3 | 在关系数据库存储上，Btree一直是主角，但在某些情况下，log(n)的读写操作并不是总是让人满意。 Bitcask是一种连续写入很快速的Key/Value数据存储结构，读写操作的时间复杂度近似常量。 4 | 5 | ####Bitcask连续写入操作 6 | Bitcask具有高效的连续写入操作，连续写操作类似向log文件追加记录，因此Bitcask也被称作是日志结构存储。 7 | 8 | Bitcask将存储对象的key、value分别存储： 9 | * 在内存中对key创建索引 10 | * 磁盘文件存储value数据 11 | 12 | 当有数据需要写入时，磁盘无需遍历文件，直接写入到数据块或者文件的末尾，避免了磁盘机械查找的时间，写入磁盘之后，只需要在内存的HashMap中更新相应的索引,内存中用HashMap来保存一条记录的索引部分，一条索引包含的信息如下： 13 | 14 | * [Key: Jason, Filename: employee.db, Offset:0, Size:146, ModifiedDate:2343432312] 15 | 16 | * [Key: Bill, Filename: employee.db, Offset:146, Size:146, ModifiedDate:5489354345] 17 | 18 | Key表示一条记录的主键，查找通过它在HashMap中找到完整索引信息 19 | Filename是磁盘文件名字，通过它和Offset找到Value在磁盘的开始位置 20 | Offset是Value在文件中偏移量，通过它和Size可以读取一条记录 21 | Size是Value所占的磁盘大小，单位是Byte 22 | 假设目前数据库中已有上述两条的记录，当我要写入key为 "Jobs"， value为: object的一条新记录时， 只需要在文件employee.db的末尾写入value=object，在HashMap中添加索引：[Key: Jobs, Filename: employee.db, Offset:292, Size:146, ModifiedDate:9489354343] 即可。 23 | 24 | 最后数据库就包含了三条索引信息： 25 | 26 | * [Key: Jason, Filename: employee.db, Offset:0, Size:146, ModifiedDate:2343432312] 27 | * [Key: Bill, Filename: employee.db, Offset:146, Size:150, ModifiedDate:5489354345] 28 | * [Key: Jobs, Filename: employee.db, Offset:294, Size:136, ModifiedDate:948965443] 29 | 30 | ####Bitcask随机读取操作 31 | 32 | 由于数据在内存当中使用HashMap作为索引，查找索引的时间为Hash查找的时间，近似常量。比如查找Bill，直接通过Key就可以找到它的索引信息，再根据索引信息，找到value在文件位置和大小，精确读取出bytes，反序列化成value对象。 当然在value存入文件时需要序列化内存对象成bytes。磁盘读取的过程的时间复杂度也是常量， 并不会随时数据的增大而增大。 33 | 34 | ####Bitcask 数据删除和更新 35 | 一条记录包含了索引和数据两个部分,删除索引容易，但要彻底的删除数据不是件容易的事情（参考磁盘空间整理）。对于更新数据，Bitcask通常采用的策略是append一条新数据，并更新已有的索引，至于旧数据则在清理数据的时候把它删除掉。 36 | 37 | ####Bitcask适合的场景 38 | * 适合连续写入，随机的读取，连续读取性能不如Btree； 39 | * 记录的key可以完全的载入内存； 40 | * value的大小比key大很多,否则意义不大； 41 | -------------------------------------------------------------------------------- /src/main/java/com/pwc/bitcask/BitCask.java: -------------------------------------------------------------------------------- 1 | package com.pwc.bitcask; 2 | 3 | import com.pwc.object.Serialization; 4 | 5 | import java.io.FileNotFoundException; 6 | import java.io.IOException; 7 | import java.io.RandomAccessFile; 8 | import java.util.HashMap; 9 | 10 | public class BitCask { 11 | private final Indexer indexer; 12 | private final String name; 13 | private static HashMap bitCasks = new HashMap<>(); 14 | private long offset; 15 | 16 | public static BitCask of(String name) { 17 | return of(name, defaultIndexOf(name)); 18 | } 19 | 20 | public static BitCask of(String name, String indexFile) { 21 | if (bitCasks.containsKey(name)) { 22 | return (BitCask) bitCasks.get(name); 23 | } else { 24 | BitCask newBitCask = new BitCask<>(name, indexFile); 25 | bitCasks.put(name, newBitCask); 26 | return newBitCask; 27 | } 28 | } 29 | 30 | public void put(String key, T value) { 31 | byte[] bytes = convertObjectToBytes(value); 32 | if (appendValue(key, bytes)) { 33 | updateIndex(key, bytes, this.offset); 34 | } 35 | } 36 | 37 | public void dumpIndexTo(String indexFile) { 38 | RandomAccessFile file = getFileAccesser(indexFile); 39 | if (file != null) { 40 | try { 41 | file.write(convertObjectToBytes(this.indexer)); 42 | } catch (IOException e) { 43 | e.printStackTrace(); 44 | } finally { 45 | if (file != null) { 46 | try { 47 | file.close(); 48 | } catch (IOException e) { 49 | e.printStackTrace(); 50 | } 51 | } 52 | } 53 | } 54 | } 55 | 56 | public T get(String key) { 57 | return readFromFile(this.indexer.get(key)); 58 | } 59 | 60 | private static String defaultIndexOf(String name) { 61 | return name + ".index"; 62 | } 63 | 64 | private BitCask(String name, String indexFile) { 65 | this.name = name; 66 | this.indexer = loadIndexFrom(indexFile); 67 | } 68 | 69 | 70 | private Indexer loadIndexFrom(String indexFile) { 71 | RandomAccessFile file = getFileAccesser(indexFile); 72 | try { 73 | if (file.length() == 0) { 74 | return new Indexer(); 75 | } 76 | } catch (IOException e) { 77 | e.printStackTrace(); 78 | } 79 | byte[] bytes = null; 80 | try { 81 | bytes = readBytesFromFile(0, (int) file.length(), indexFile); 82 | } catch (IOException e) { 83 | e.printStackTrace(); 84 | } finally { 85 | if (file != null) { 86 | try { 87 | file.close(); 88 | } catch (IOException e) { 89 | e.printStackTrace(); 90 | } 91 | } 92 | } 93 | 94 | if (bytes == null) { 95 | return new Indexer(); 96 | } 97 | Object object = convertBytesToObject(bytes); 98 | if (!(object instanceof Indexer)) 99 | return new Indexer(); 100 | else 101 | return (Indexer) object; 102 | } 103 | 104 | 105 | private boolean appendValue(String key, byte[] bytes) { 106 | return appendBytesToFile(bytes, this.name); 107 | } 108 | 109 | private boolean appendBytesToFile(byte[] bytes, String name) { 110 | RandomAccessFile file = getFileAccesser(name); 111 | try { 112 | long offset = file.length(); 113 | file.seek(offset); 114 | file.write(bytes); 115 | this.offset = offset; 116 | return true; 117 | } catch (IOException e) { 118 | e.printStackTrace(); 119 | } finally { 120 | if (file != null) { 121 | try { 122 | file.close(); 123 | } catch (IOException e) { 124 | e.printStackTrace(); 125 | } 126 | } 127 | 128 | } 129 | return false; 130 | } 131 | 132 | 133 | private RandomAccessFile getFileAccesser(String name) { 134 | try { 135 | return new RandomAccessFile(name, "rw"); 136 | } catch (FileNotFoundException e) { 137 | e.printStackTrace(); 138 | 139 | } 140 | return null; 141 | } 142 | 143 | private byte[] convertObjectToBytes(Object value) { 144 | try { 145 | return Serialization.serialize(value); 146 | } catch (IOException e) { 147 | e.printStackTrace(); 148 | return null; 149 | } 150 | } 151 | 152 | private void updateIndex(String key, byte[] bytes, long offset) { 153 | this.indexer.put(key, new Index(key, this.name, offset, bytes.length)); 154 | } 155 | 156 | 157 | private T readFromFile(Index index) { 158 | byte[] bytes = readBytesFromFile(index.offset, index.size, index.fileName); 159 | Object object = convertBytesToObject(bytes); 160 | return (T) object; 161 | } 162 | 163 | private byte[] readBytesFromFile(long offset, int size, String fileName) { 164 | byte[] bytes = new byte[size]; 165 | RandomAccessFile file = getFileAccesser(fileName); 166 | try { 167 | file.seek(offset); 168 | file.read(bytes); 169 | } catch (IOException e) { 170 | e.printStackTrace(); 171 | bytes = null; 172 | } finally { 173 | if (file != null) { 174 | try { 175 | file.close(); 176 | } catch (IOException e) { 177 | e.printStackTrace(); 178 | } 179 | } 180 | } 181 | return bytes; 182 | } 183 | 184 | private Object convertBytesToObject(byte[] bytes) { 185 | try { 186 | return Serialization.deserialize(bytes); 187 | } catch (IOException e) { 188 | e.printStackTrace(); 189 | } catch (ClassNotFoundException e) { 190 | e.printStackTrace(); 191 | } 192 | return null; 193 | } 194 | 195 | } 196 | -------------------------------------------------------------------------------- /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 46 | 47 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 172 | 173 | 174 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 214 | 215 | 216 | 217 | 220 | 221 | 224 | 225 | 226 | 227 | 230 | 231 | 234 | 235 | 238 | 239 | 240 | 241 | 244 | 245 | 248 | 249 | 252 | 253 | 256 | 257 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 354 | 355 | 362 | 363 | 364 | 382 | 389 | 390 | 391 | 402 | 403 | 416 | 417 | 418 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | localhost 448 | 5050 449 | 450 | 451 | 452 | 453 | 454 | 455 | 1380429000588 456 | 1380429000588 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 489 | 490 | 501 | 543 | 544 | 545 | 546 | 547 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 579 | 580 | 581 | 582 | 583 | 584 | 585 | 586 | 587 | 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 | 604 | 605 | 606 | 607 | 608 | 609 | 610 | 611 | 612 | 613 | 614 | 615 | 616 | 617 | 618 | 619 | 620 | 621 | 622 | 623 | 624 | 625 | 626 | 627 | 628 | 629 | 630 | 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | 640 | 641 | 642 | 643 | 644 | 649 | 650 | 651 | 652 | 653 | 654 | No facets are configured 655 | 656 | 661 | 662 | 663 | 664 | 665 | 666 | 667 | 672 | 673 | 674 | 675 | 676 | 677 | 1.8 678 | 679 | 684 | 685 | 686 | 687 | 688 | 689 | BitCask 690 | 691 | 697 | 698 | 699 | 700 | 701 | 702 | 703 | 708 | 709 | 710 | 711 | 712 | 713 | 714 | 715 | --------------------------------------------------------------------------------