├── .gitignore ├── LICENSE ├── README.md ├── TODO.md ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── jordanwilliams │ │ └── heftydb │ │ ├── cache │ │ └── TableBlockCache.java │ │ ├── compact │ │ ├── CompactionPlan.java │ │ ├── CompactionStrategies.java │ │ ├── CompactionStrategy.java │ │ ├── CompactionTables.java │ │ ├── CompactionTask.java │ │ ├── Compactor.java │ │ └── planner │ │ │ ├── CompactionPlanner.java │ │ │ ├── FullCompactionPlanner.java │ │ │ └── SizeTieredCompactionPlanner.java │ │ ├── data │ │ ├── Key.java │ │ ├── Tuple.java │ │ └── Value.java │ │ ├── db │ │ ├── Config.java │ │ ├── DB.java │ │ ├── DBInitializer.java │ │ ├── DBState.java │ │ ├── HeftyDB.java │ │ ├── Record.java │ │ └── Snapshot.java │ │ ├── index │ │ ├── Index.java │ │ ├── IndexBlock.java │ │ ├── IndexRecord.java │ │ └── IndexWriter.java │ │ ├── io │ │ ├── AppendChannelFile.java │ │ ├── AppendFile.java │ │ ├── ImmutableChannelFile.java │ │ ├── ImmutableFile.java │ │ └── Throttle.java │ │ ├── metrics │ │ ├── CacheHitGauge.java │ │ └── Metrics.java │ │ ├── offheap │ │ ├── BitSet.java │ │ ├── BloomFilter.java │ │ ├── JVMUnsafe.java │ │ ├── MemoryAllocator.java │ │ ├── MemoryPointer.java │ │ ├── Offheap.java │ │ ├── SortedByteMap.java │ │ └── allocator │ │ │ ├── Allocator.java │ │ │ └── UnsafeAllocator.java │ │ ├── read │ │ ├── CompactionTupleIterator.java │ │ ├── LatestTupleIterator.java │ │ ├── MergingIterator.java │ │ ├── TableAggregationIterator.java │ │ └── TableReader.java │ │ ├── state │ │ ├── Caches.java │ │ ├── Paths.java │ │ ├── Snapshots.java │ │ └── Tables.java │ │ ├── table │ │ ├── MutableTable.java │ │ ├── Table.java │ │ ├── file │ │ │ ├── FileTable.java │ │ │ ├── FileTableWriter.java │ │ │ ├── TableBloomFilter.java │ │ │ ├── TableBloomFilterWriter.java │ │ │ ├── TableTrailer.java │ │ │ └── TupleBlock.java │ │ └── memory │ │ │ ├── MemoryTable.java │ │ │ ├── SkipListTupleMap.java │ │ │ ├── SortedTupleMap.java │ │ │ └── SynchronizedTupleMap.java │ │ ├── util │ │ ├── ByteBuffers.java │ │ ├── CloseableIterator.java │ │ ├── MurmurHash3.java │ │ ├── PeekableIterator.java │ │ ├── Serializer.java │ │ ├── Sizes.java │ │ └── XORShiftRandom.java │ │ └── write │ │ ├── CommitLog.java │ │ ├── CommitLogWriter.java │ │ └── TableWriter.java └── resources │ └── logback.xml └── test └── java └── com └── jordanwilliams └── heftydb └── test ├── base ├── FileTest.java ├── ParameterizedIntegrationTest.java ├── ParameterizedTupleTest.java └── TupleTest.java ├── endurance └── EnduranceTest.java ├── generator ├── ConfigGenerator.java ├── KeyValueGenerator.java └── TupleGenerator.java ├── helper ├── CompareHelper.java ├── PerformanceHelper.java ├── StopWatch.java └── TestFileHelper.java ├── integration ├── ConcurrencyTest.java ├── DeleteTest.java ├── IteratorTest.java └── ReadWriteTest.java ├── performance ├── db │ ├── ReadPerformance.java │ ├── ReadWritePerformance.java │ ├── ScanPerformance.java │ └── WritePerformance.java ├── offheap │ ├── BlockCreationPerformance.java │ └── MemoryPerformance.java ├── table │ ├── file │ │ ├── FileTablePerformance.java │ │ ├── IndexBlockPerformance.java │ │ ├── IndexPerformance.java │ │ ├── RecordBlockPerformance.java │ │ └── TableBloomFilterPerformance.java │ └── memory │ │ └── MemoryTablePerformance.java └── write │ └── RecordWriterPerformance.java └── unit ├── data └── TupleTest.java ├── db └── DBInitializerTest.java ├── index ├── IndexBlockRandomTest.java ├── IndexBlockTest.java └── IndexTest.java ├── io └── FileIOTest.java ├── offheap ├── BitSetTest.java ├── BloomFilterTest.java ├── MemoryPointerTest.java ├── SortedByteMapTest.java └── UnsafeAllocatorTest.java ├── read ├── CompactionTupleIteratorTest.java ├── LatestTupleIteratorTest.java ├── MergingIteratorTest.java └── TableAggregationIteratorTest.java ├── table ├── file │ ├── FileTableTest.java │ ├── TableBloomFilterTest.java │ ├── TupleBlockRandomTest.java │ └── TupleBlockTest.java └── memory │ └── MemoryTableTest.java ├── util └── XORShiftRandomTest.java └── write └── CommitLogTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | # Package Files # 3 | *.jar 4 | *.war 5 | *.ear 6 | .idea 7 | *.iml 8 | target 9 | *.log -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HeftyDB 2 | 3 | *"You put your data in it."* 4 | 5 | HeftyDB is a persistent, sorted, key-value library for the JVM. It was designed with the following goals in mind: 6 | 7 | 1. Be as fast and memory efficient as is feasible on the JVM for random reads, random writes, and range scans. 8 | 2. Provide a stable base on which to build new and interesting storage systems. 9 | 3. Provide detailed metrics about what is going on at every level of the stack. 10 | 4. Have clean, understandable code that others can learn from. 11 | 12 | *Note: HeftyDB was built primarily for fun and learning. While the code is generally production quality and has extensive test coverage, you probably shouldn't use it in production unless you know what you are doing. So, yea, don't use it in production and lose a bunch of important data, or something.* 13 | 14 | ## Features 15 | 16 | ### Simple API 17 | Supports gets and puts by key as well as ascending and descending iteration from any key. 18 | 19 | ### Log Structured Merge Trees 20 | All write operations are sequential, and are limited by the sequential IO performance of the underlying disk. Tables 21 | are written with a full B+tree index for memory efficiency, even at very large table sizes. 22 | 23 | ### Snapshotted multi-version concurrency control 24 | Reads and range scans never block writes. 25 | 26 | ### Off-heap data structures 27 | Operations in the critical read and write paths are implemented using off-heap memory wherever possible to reduce GC 28 | pressure and memory overhead. 29 | 30 | ### Multi-threaded compactions 31 | Make use of multiple CPU cores for table writes and table compactions 32 | 33 | ### Pluggable compaction strategies 34 | Provide custom compaction behavior tailored to specific workloads. 35 | 36 | ## Design Details 37 | https://github.com/jordw/heftydb/wiki/Design-Overview 38 | 39 | ## Example Usage 40 | 41 | ```java 42 | try { 43 | //Open a HeftyDB in a directory 44 | DB testDB = HeftyDB.open(new Config.Builder().directory(directory).build()); 45 | 46 | //Write a key 47 | Snapshot snapshot = testDB.put(someByteBufferKey, someByteBufferValue); 48 | 49 | //Read a key at a particular snapshot 50 | Record record = testDB.get(someByteBufferKey, snapshot); 51 | 52 | //Delete a key 53 | Snapshot deleteSnapshot = testDB.delete(someByteBufferKey); 54 | 55 | //Get an ascending iterator of keys greater than or equal 56 | //to the provided key at the provided snapshot 57 | CloseableIterator ascendingIterator = testDB.ascendingIterator(someByteBufferKey, snapshot); 58 | 59 | while (ascendingIterator.hasNext()) { 60 | Record next = ascendingIterator.next(); 61 | } 62 | 63 | //Get a descending iterator of keys less than or equal 64 | //to the provided key at the provided snapshot 65 | CloseableIterator descendingIterator = testDB.descendingIterator(someByteBufferKey, snapshot); 66 | 67 | while (descendingIterator.hasNext()) { 68 | Record next = descendingIterator.next(); 69 | } 70 | 71 | //Compact the database 72 | Future compactionFuture = testDB.compact(); 73 | compactionFuture.get(); 74 | 75 | //Close the database 76 | testDB.close(); 77 | } catch (IOException e){ 78 | Logger.error(e); 79 | } 80 | ``` 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | #To Do 2 | 3 | Add additional javadoc -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | heftydb 8 | heftydb 9 | 0.1-SNAPSHOT 10 | 11 | 12 | 13 | com.google.guava 14 | guava 15 | 16.0 16 | 17 | 18 | com.googlecode.concurrentlinkedhashmap 19 | concurrentlinkedhashmap-lru 20 | 1.4 21 | 22 | 23 | com.codahale.metrics 24 | metrics-core 25 | 3.0.1 26 | 27 | 28 | org.slf4j 29 | slf4j-api 30 | 1.6.4 31 | 32 | 33 | ch.qos.logback 34 | logback-classic 35 | 1.0.13 36 | 37 | 38 | ch.qos.logback 39 | logback-core 40 | 1.0.13 41 | 42 | 43 | org.apache.commons 44 | commons-lang3 45 | 3.0 46 | test 47 | 48 | 49 | junit 50 | junit 51 | 4.13.1 52 | test 53 | 54 | 55 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/CompactionPlan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact; 18 | 19 | import java.util.Arrays; 20 | import java.util.Iterator; 21 | import java.util.List; 22 | 23 | /** 24 | * A collection of CompactionTasks that can execute in parallel. 25 | */ 26 | public class CompactionPlan implements Iterable { 27 | 28 | private final List tasks; 29 | 30 | public CompactionPlan(List tasks) { 31 | this.tasks = tasks; 32 | } 33 | 34 | public CompactionPlan(CompactionTask... tasks) { 35 | this(Arrays.asList(tasks)); 36 | } 37 | 38 | public List tasks() { 39 | return tasks; 40 | } 41 | 42 | @Override 43 | public boolean equals(Object o) { 44 | if (this == o) return true; 45 | if (o == null || getClass() != o.getClass()) return false; 46 | 47 | CompactionPlan that = (CompactionPlan) o; 48 | 49 | if (tasks != null ? !tasks.equals(that.tasks) : that.tasks != null) return false; 50 | 51 | return true; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | return tasks != null ? tasks.hashCode() : 0; 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return "CompactionPlan{" + 62 | "tasks=" + tasks + 63 | '}'; 64 | } 65 | 66 | @Override 67 | public Iterator iterator() { 68 | return tasks.iterator(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/CompactionStrategies.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact; 18 | 19 | import com.jordanwilliams.heftydb.compact.planner.CompactionPlanner; 20 | import com.jordanwilliams.heftydb.compact.planner.FullCompactionPlanner; 21 | import com.jordanwilliams.heftydb.compact.planner.SizeTieredCompactionPlanner; 22 | 23 | import java.util.Collections; 24 | 25 | /** 26 | * Contains built in compaction strategies. 27 | */ 28 | public enum CompactionStrategies implements CompactionStrategy { 29 | 30 | SIZE_TIERED_COMPACTION_STRATEGY { 31 | @Override 32 | public CompactionPlanner initialize(CompactionTables tables) { 33 | return new SizeTieredCompactionPlanner(tables); 34 | } 35 | }, 36 | 37 | NULL_COMPACTION_STRATEGY { 38 | @Override 39 | public CompactionPlanner initialize(CompactionTables tables) { 40 | return new CompactionPlanner() { 41 | @Override 42 | public CompactionPlan planCompaction() { 43 | return new CompactionPlan(Collections.emptyList()); 44 | } 45 | 46 | @Override 47 | public boolean needsCompaction() { 48 | return false; 49 | } 50 | }; 51 | } 52 | }, 53 | 54 | FULL_COMPACTION_STRATEGY { 55 | @Override 56 | public CompactionPlanner initialize(CompactionTables tables) { 57 | return new FullCompactionPlanner(tables); 58 | } 59 | }; 60 | 61 | @Override 62 | public abstract CompactionPlanner initialize(CompactionTables tables); 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/CompactionStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact; 18 | 19 | import com.jordanwilliams.heftydb.compact.planner.CompactionPlanner; 20 | 21 | /** 22 | * Represents a particular CompactionPlanner. 23 | */ 24 | public interface CompactionStrategy { 25 | 26 | public CompactionPlanner initialize(CompactionTables tables); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/CompactionTables.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact; 18 | 19 | import com.jordanwilliams.heftydb.state.Tables; 20 | import com.jordanwilliams.heftydb.table.Table; 21 | 22 | import java.util.ArrayList; 23 | import java.util.HashSet; 24 | import java.util.List; 25 | import java.util.Set; 26 | 27 | /** 28 | * Wraps a set of Tables and tracks compaction status for each table. 29 | */ 30 | public class CompactionTables { 31 | 32 | private final Set alreadyCompactedTables = new HashSet(); 33 | private final Tables tables; 34 | 35 | public CompactionTables(Tables tables) { 36 | this.tables = tables; 37 | } 38 | 39 | public List eligibleTables() { 40 | List
eligibleTables = new ArrayList
(); 41 | 42 | tables.readLock(); 43 | 44 | try { 45 | for (Table table : tables){ 46 | if (table.isPersistent() && !alreadyCompactedTables.contains(table.id())){ 47 | eligibleTables.add(table); 48 | } 49 | } 50 | 51 | } finally { 52 | tables.readUnlock(); 53 | } 54 | 55 | return eligibleTables; 56 | } 57 | 58 | public void markAsCompacted(Table table){ 59 | alreadyCompactedTables.add(table.id()); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/CompactionTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact; 18 | 19 | import com.jordanwilliams.heftydb.table.Table; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | /** 25 | * Represents a single CompactionTask. 26 | */ 27 | public class CompactionTask { 28 | 29 | public enum Priority { 30 | HIGH, NORMAL 31 | } 32 | 33 | public static class Builder { 34 | 35 | private final List
tables = new ArrayList
(); 36 | private final int level; 37 | private final Priority priority; 38 | 39 | public Builder(int level, Priority priority) { 40 | this.level = level; 41 | this.priority = priority; 42 | } 43 | 44 | public void add(Table table) { 45 | tables.add(table); 46 | } 47 | 48 | public CompactionTask build() { 49 | return new CompactionTask(tables, level, priority); 50 | } 51 | } 52 | 53 | private final List
tables; 54 | private final int level; 55 | private final Priority priority; 56 | 57 | public CompactionTask(List
tables, int level, Priority priority) { 58 | this.tables = tables; 59 | this.level = level; 60 | this.priority = priority; 61 | } 62 | 63 | public List
tables() { 64 | return tables; 65 | } 66 | 67 | public int level() { 68 | return level; 69 | } 70 | 71 | public Priority priority() { 72 | return priority; 73 | } 74 | 75 | @Override 76 | public boolean equals(Object o) { 77 | if (this == o) return true; 78 | if (o == null || getClass() != o.getClass()) return false; 79 | 80 | CompactionTask that = (CompactionTask) o; 81 | 82 | if (level != that.level) return false; 83 | if (priority != that.priority) return false; 84 | if (tables != null ? !tables.equals(that.tables) : that.tables != null) return false; 85 | 86 | return true; 87 | } 88 | 89 | @Override 90 | public int hashCode() { 91 | int result = tables != null ? tables.hashCode() : 0; 92 | result = 31 * result + level; 93 | result = 31 * result + (priority != null ? priority.hashCode() : 0); 94 | return result; 95 | } 96 | 97 | @Override 98 | public String toString() { 99 | return "CompactionTask{" + 100 | "tables=" + tables + 101 | ", level=" + level + 102 | ", priority=" + priority + 103 | '}'; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/planner/CompactionPlanner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact.planner; 18 | 19 | import com.jordanwilliams.heftydb.compact.CompactionPlan; 20 | 21 | public interface CompactionPlanner { 22 | 23 | public CompactionPlan planCompaction(); 24 | 25 | public boolean needsCompaction(); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/planner/FullCompactionPlanner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact.planner; 18 | 19 | import com.jordanwilliams.heftydb.compact.CompactionPlan; 20 | import com.jordanwilliams.heftydb.compact.CompactionTables; 21 | import com.jordanwilliams.heftydb.compact.CompactionTask; 22 | import com.jordanwilliams.heftydb.table.Table; 23 | 24 | import java.util.List; 25 | 26 | /** 27 | * Compacts all tables into a single table. 28 | */ 29 | public class FullCompactionPlanner implements CompactionPlanner { 30 | 31 | private final CompactionTables tables; 32 | 33 | public FullCompactionPlanner(CompactionTables tables) { 34 | this.tables = tables; 35 | } 36 | 37 | @Override 38 | public CompactionPlan planCompaction() { 39 | CompactionTask.Builder taskBuilder = new CompactionTask.Builder(2, CompactionTask.Priority.NORMAL); 40 | 41 | List
eligibleTables = tables.eligibleTables(); 42 | 43 | if (eligibleTables.size() > 1) { 44 | for (Table table : eligibleTables){ 45 | taskBuilder.add(table); 46 | } 47 | 48 | return new CompactionPlan(taskBuilder.build()); 49 | } 50 | 51 | return null; 52 | } 53 | 54 | @Override 55 | public boolean needsCompaction() { 56 | return false; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/compact/planner/SizeTieredCompactionPlanner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.compact.planner; 18 | 19 | import com.jordanwilliams.heftydb.compact.CompactionPlan; 20 | import com.jordanwilliams.heftydb.compact.CompactionTables; 21 | import com.jordanwilliams.heftydb.compact.CompactionTask; 22 | import com.jordanwilliams.heftydb.table.Table; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.SortedMap; 28 | import java.util.TreeMap; 29 | 30 | /** 31 | * Compacts tables into groups with overlapping key ranges once there are 10 tables at a particular size. 32 | */ 33 | public class SizeTieredCompactionPlanner implements CompactionPlanner { 34 | 35 | private static final int MAX_LEVEL_TABLES = 5; 36 | 37 | private final CompactionTables tables; 38 | 39 | public SizeTieredCompactionPlanner(CompactionTables tables) { 40 | this.tables = tables; 41 | } 42 | 43 | @Override 44 | public CompactionPlan planCompaction() { 45 | SortedMap> leveledTables = leveledTables(); 46 | List compactionTasks = new ArrayList(); 47 | 48 | for (Map.Entry> entry : leveledTables.entrySet()) { 49 | if (entry.getValue().size() >= MAX_LEVEL_TABLES) { 50 | int level = entry.getKey(); 51 | compactionTasks.add(new CompactionTask(entry.getValue(), level + 1, level < 3 ? CompactionTask 52 | .Priority.HIGH : CompactionTask.Priority.NORMAL)); 53 | } 54 | } 55 | 56 | return new CompactionPlan(compactionTasks); 57 | } 58 | 59 | @Override 60 | public boolean needsCompaction() { 61 | SortedMap> leveledTables = leveledTables(); 62 | 63 | for (Map.Entry> entry : leveledTables.entrySet()) { 64 | if (entry.getValue().size() >= MAX_LEVEL_TABLES) { 65 | return true; 66 | } 67 | } 68 | 69 | return false; 70 | } 71 | 72 | private SortedMap> leveledTables() { 73 | SortedMap> tableMap = new TreeMap>(); 74 | 75 | List
eligibleTables = tables.eligibleTables(); 76 | 77 | for (Table table : eligibleTables) { 78 | List
levelTables = tableMap.get(table.level()); 79 | 80 | if (levelTables == null) { 81 | levelTables = new ArrayList
(); 82 | tableMap.put(table.level(), levelTables); 83 | } 84 | 85 | levelTables.add(table); 86 | } 87 | 88 | return tableMap; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/data/Key.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.data; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | /** 22 | * Encapsulates a database Key. Contains both a ByteBuffer with the Key data as well as a long that represents the 23 | * snapshot that the Key was written at. 24 | */ 25 | public class Key implements Comparable { 26 | 27 | private final ByteBuffer data; 28 | private final long snapshotId; 29 | 30 | public Key(ByteBuffer data, long snapshotId) { 31 | this.data = data; 32 | this.snapshotId = snapshotId; 33 | } 34 | 35 | public ByteBuffer data() { 36 | return data; 37 | } 38 | 39 | public long snapshotId() { 40 | return snapshotId; 41 | } 42 | 43 | public int size() { 44 | return data.capacity(); 45 | } 46 | 47 | @Override 48 | public int compareTo(Key o) { 49 | int compared = data.compareTo(o.data); 50 | 51 | if (compared != 0) { 52 | return compared; 53 | } 54 | 55 | return Long.compare(snapshotId, o.snapshotId); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object o) { 60 | if (this == o) return true; 61 | if (o == null || getClass() != o.getClass()) return false; 62 | 63 | Key key = (Key) o; 64 | 65 | if (snapshotId != key.snapshotId) return false; 66 | if (data != null ? !data.equals(key.data) : key.data != null) return false; 67 | 68 | return true; 69 | } 70 | 71 | @Override 72 | public int hashCode() { 73 | return data != null ? data.hashCode() : 0; 74 | } 75 | 76 | @Override 77 | public String toString() { 78 | byte[] keyArray = new byte[data.capacity()]; 79 | data.rewind(); 80 | data.get(keyArray); 81 | data.rewind(); 82 | 83 | return "Key{" + 84 | "data=" + new String(keyArray) + 85 | ", snapshotId=" + snapshotId + 86 | '}'; 87 | } 88 | } -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/data/Value.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.data; 18 | 19 | import com.jordanwilliams.heftydb.util.ByteBuffers; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * A wrapper class around a ByteBuffer that encapsulates a database Value. 25 | */ 26 | public class Value implements Comparable { 27 | 28 | public static Value TOMBSTONE_VALUE = new Value(ByteBuffers.EMPTY_BUFFER); 29 | 30 | private final ByteBuffer value; 31 | 32 | public Value(ByteBuffer value) { 33 | this.value = value; 34 | } 35 | 36 | public ByteBuffer data() { 37 | return value; 38 | } 39 | 40 | public boolean isEmpty() { 41 | return value.capacity() == 0; 42 | } 43 | 44 | public int size() { 45 | return value.capacity(); 46 | } 47 | 48 | @Override 49 | public int compareTo(Value o) { 50 | return value.compareTo(o.value); 51 | } 52 | 53 | @Override 54 | public boolean equals(Object o) { 55 | if (this == o) { 56 | return true; 57 | } 58 | if (o == null || getClass() != o.getClass()) { 59 | return false; 60 | } 61 | 62 | Value value1 = (Value) o; 63 | 64 | if (value != null ? !value.equals(value1.value) : value1.value != null) { 65 | return false; 66 | } 67 | 68 | return true; 69 | } 70 | 71 | @Override 72 | public int hashCode() { 73 | return value != null ? value.hashCode() : 0; 74 | } 75 | 76 | @Override 77 | public String toString() { 78 | return "Value{" + 79 | "data=" + new String(value.array()) + 80 | "}"; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/db/DB.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.db; 18 | 19 | import com.jordanwilliams.heftydb.util.CloseableIterator; 20 | 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.util.concurrent.Future; 24 | 25 | /** 26 | * The public API for a database implementation. 27 | */ 28 | public interface DB { 29 | 30 | public Snapshot put(ByteBuffer key, ByteBuffer value) throws IOException; 31 | 32 | public Snapshot put(ByteBuffer key, ByteBuffer value, boolean fsync) throws IOException; 33 | 34 | public Record get(ByteBuffer key) throws IOException; 35 | 36 | public Record get(ByteBuffer key, Snapshot snapshot) throws IOException; 37 | 38 | public Snapshot delete(ByteBuffer key) throws IOException; 39 | 40 | public CloseableIterator ascendingIterator(Snapshot snapshot) throws IOException; 41 | 42 | public CloseableIterator ascendingIterator(ByteBuffer key, Snapshot snapshot) throws IOException; 43 | 44 | public CloseableIterator descendingIterator(Snapshot snapshot) throws IOException; 45 | 46 | public CloseableIterator descendingIterator(ByteBuffer key, Snapshot snapshot) throws IOException; 47 | 48 | public void retainSnapshot(Snapshot snapshot); 49 | 50 | public void releaseSnapshot(Snapshot snapshot); 51 | 52 | public void close() throws IOException; 53 | 54 | public void logMetrics(); 55 | 56 | public Future compact() throws IOException; 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/db/DBState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.db; 18 | 19 | import com.jordanwilliams.heftydb.state.Caches; 20 | import com.jordanwilliams.heftydb.state.Paths; 21 | import com.jordanwilliams.heftydb.state.Snapshots; 22 | import com.jordanwilliams.heftydb.state.Tables; 23 | import com.jordanwilliams.heftydb.table.Table; 24 | 25 | import java.util.Collection; 26 | 27 | /** 28 | * Represents the state of a database. 29 | */ 30 | public class DBState { 31 | 32 | private final Tables tables; 33 | private final Snapshots snapshots; 34 | private final Config config; 35 | private final Paths paths; 36 | private final Caches caches; 37 | 38 | public DBState(Collection
tables, Config config, Paths paths, Caches caches, long currentSnapshotId) { 39 | this.snapshots = new Snapshots(currentSnapshotId); 40 | this.tables = new Tables(tables); 41 | this.config = config; 42 | this.paths = paths; 43 | this.caches = caches; 44 | } 45 | 46 | public Paths paths() { 47 | return paths; 48 | } 49 | 50 | public Tables tables() { 51 | return tables; 52 | } 53 | 54 | public Snapshots snapshots() { 55 | return snapshots; 56 | } 57 | 58 | public Caches caches() { 59 | return caches; 60 | } 61 | 62 | public Config config() { 63 | return config; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/db/Record.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.db; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.util.ByteBuffers; 21 | import com.jordanwilliams.heftydb.util.CloseableIterator; 22 | 23 | import java.io.IOException; 24 | import java.nio.ByteBuffer; 25 | 26 | /** 27 | * A public wrapper class that represents the results of a database read. 28 | */ 29 | public class Record implements Comparable { 30 | 31 | public static class TupleIterator implements CloseableIterator { 32 | 33 | private final CloseableIterator tupleIterator; 34 | 35 | public TupleIterator(CloseableIterator tupleIterator) { 36 | this.tupleIterator = tupleIterator; 37 | } 38 | 39 | @Override 40 | public boolean hasNext() { 41 | return tupleIterator.hasNext(); 42 | } 43 | 44 | @Override 45 | public Record next() { 46 | return new Record(tupleIterator.next()); 47 | } 48 | 49 | @Override 50 | public void remove() { 51 | tupleIterator.remove(); 52 | } 53 | 54 | @Override 55 | public void close() throws IOException { 56 | tupleIterator.close(); 57 | } 58 | } 59 | 60 | private final ByteBuffer key; 61 | private final ByteBuffer value; 62 | private final Snapshot snapshot; 63 | 64 | public Record(ByteBuffer key, ByteBuffer value, Snapshot snapshot) { 65 | this.key = key; 66 | this.value = value; 67 | this.snapshot = snapshot; 68 | } 69 | 70 | public Record(Tuple tuple) { 71 | this(tuple.key().data(), tuple.value().data(), new Snapshot(tuple.key().snapshotId())); 72 | } 73 | 74 | public ByteBuffer key() { 75 | return key; 76 | } 77 | 78 | public ByteBuffer value() { 79 | return value; 80 | } 81 | 82 | public Snapshot snapshot() { 83 | return snapshot; 84 | } 85 | 86 | @Override 87 | public int compareTo(Record o) { 88 | int compare = key.compareTo(o.key); 89 | 90 | if (compare != 0) { 91 | return 0; 92 | } 93 | 94 | return Long.compare(snapshot.id(), o.snapshot.id()); 95 | } 96 | 97 | @Override 98 | public boolean equals(Object o) { 99 | if (this == o) return true; 100 | if (o == null || getClass() != o.getClass()) return false; 101 | 102 | Record record = (Record) o; 103 | 104 | if (key != null ? !key.equals(record.key) : record.key != null) return false; 105 | if (snapshot != null ? !snapshot.equals(record.snapshot) : record.snapshot != null) return false; 106 | if (value != null ? !value.equals(record.value) : record.value != null) return false; 107 | 108 | return true; 109 | } 110 | 111 | @Override 112 | public int hashCode() { 113 | int result = key != null ? key.hashCode() : 0; 114 | result = 31 * result + (value != null ? value.hashCode() : 0); 115 | result = 31 * result + (snapshot != null ? snapshot.hashCode() : 0); 116 | return result; 117 | } 118 | 119 | @Override 120 | public String toString() { 121 | return "Record{" + 122 | "key=" + ByteBuffers.toString(key) + 123 | ", value=" + ByteBuffers.toString(value) + 124 | ", snapshot=" + snapshot + 125 | '}'; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/db/Snapshot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.db; 18 | 19 | /** 20 | * Represents a database snapshot. 21 | */ 22 | public class Snapshot implements Comparable { 23 | 24 | public static final Snapshot MAX = new Snapshot(Long.MAX_VALUE); 25 | public static final Snapshot MIN = new Snapshot(0); 26 | 27 | private final long id; 28 | 29 | public Snapshot(long id) { 30 | this.id = id; 31 | } 32 | 33 | public long id() { 34 | return id; 35 | } 36 | 37 | @Override 38 | public int compareTo(Snapshot o) { 39 | return Long.compare(id, o.id()); 40 | } 41 | 42 | @Override 43 | public boolean equals(Object o) { 44 | if (this == o) { 45 | return true; 46 | } 47 | if (o == null || getClass() != o.getClass()) { 48 | return false; 49 | } 50 | 51 | Snapshot snapshot = (Snapshot) o; 52 | 53 | if (id != snapshot.id) { 54 | return false; 55 | } 56 | 57 | return true; 58 | } 59 | 60 | @Override 61 | public int hashCode() { 62 | return (int) (id ^ (id >>> 32)); 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return "Snapshot{" + 68 | "id=" + id + 69 | '}'; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/index/IndexRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.index; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.util.Sizes; 21 | 22 | /** 23 | * Represents a single entry in an IndexBlock. 24 | */ 25 | public class IndexRecord { 26 | 27 | private final Key startKey; 28 | private final long blockOffset; 29 | private final int blockSize; 30 | private final boolean isLeaf; 31 | 32 | public IndexRecord(Key startKey, long blockOffset, int blockSize, boolean isLeaf) { 33 | this.startKey = startKey; 34 | this.blockOffset = blockOffset; 35 | this.blockSize = blockSize; 36 | this.isLeaf = isLeaf; 37 | } 38 | 39 | public IndexRecord(Key startKey, long blockOffset, int blockSize) { 40 | this(startKey, blockOffset, blockSize, true); 41 | } 42 | 43 | public Key startKey() { 44 | return startKey; 45 | } 46 | 47 | public long blockOffset() { 48 | return blockOffset; 49 | } 50 | 51 | public int blockSize() { 52 | return blockSize; 53 | } 54 | 55 | public int size() { 56 | return Sizes.INT_SIZE + //Key blockSize 57 | startKey.size() + //Key 58 | Sizes.LONG_SIZE + //Offset 59 | Sizes.INT_SIZE + 60 | 1; //Leaf flag 61 | } 62 | 63 | public int contentsSize() { 64 | return Sizes.LONG_SIZE + //Offset 65 | Sizes.INT_SIZE + 66 | 1; //Leaf flag 67 | } 68 | 69 | public boolean isLeaf() { 70 | return isLeaf; 71 | } 72 | 73 | @Override 74 | public boolean equals(Object o) { 75 | if (this == o) return true; 76 | if (o == null || getClass() != o.getClass()) return false; 77 | 78 | IndexRecord that = (IndexRecord) o; 79 | 80 | if (blockOffset != that.blockOffset) return false; 81 | if (blockSize != that.blockSize) return false; 82 | if (isLeaf != that.isLeaf) return false; 83 | if (startKey != null ? !startKey.equals(that.startKey) : that.startKey != null) return false; 84 | 85 | return true; 86 | } 87 | 88 | @Override 89 | public int hashCode() { 90 | int result = startKey != null ? startKey.hashCode() : 0; 91 | result = 31 * result + (int) (blockOffset ^ (blockOffset >>> 32)); 92 | result = 31 * result + blockSize; 93 | result = 31 * result + (isLeaf ? 1 : 0); 94 | return result; 95 | } 96 | 97 | @Override 98 | public String toString() { 99 | return "IndexRecord{" + 100 | "startKey=" + startKey + 101 | ", blockOffset=" + blockOffset + 102 | ", blockSize=" + blockSize + 103 | ", isLeaf=" + isLeaf + 104 | '}'; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/io/AppendFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.io; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * An append only file. 25 | */ 26 | public interface AppendFile extends Closeable { 27 | 28 | public long append(ByteBuffer buffer) throws IOException; 29 | 30 | public long appendLong(long value) throws IOException; 31 | 32 | public long appendInt(int value) throws IOException; 33 | 34 | public long size() throws IOException; 35 | 36 | public void sync() throws IOException; 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/io/ImmutableChannelFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.io; 18 | 19 | import com.jordanwilliams.heftydb.util.Sizes; 20 | 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.channels.FileChannel; 24 | import java.nio.file.Path; 25 | import java.nio.file.StandardOpenOption; 26 | 27 | /** 28 | * A read only wrapper around a FileChannel. 29 | */ 30 | public class ImmutableChannelFile implements ImmutableFile { 31 | 32 | private static final ThreadLocal primitiveBuffer = new ThreadLocal() { 33 | @Override 34 | protected ByteBuffer initialValue() { 35 | ByteBuffer primitiveBuffer = ByteBuffer.allocate(Sizes.LONG_SIZE); 36 | return primitiveBuffer; 37 | } 38 | }; 39 | 40 | private final FileChannel channel; 41 | 42 | public ImmutableChannelFile(FileChannel channel) { 43 | this.channel = channel; 44 | } 45 | 46 | @Override 47 | public long read(ByteBuffer bufferToRead, long position) throws IOException { 48 | return channel.read(bufferToRead, position); 49 | } 50 | 51 | @Override 52 | public int readInt(long position) throws IOException { 53 | ByteBuffer intBuffer = intBuffer(); 54 | channel.read(intBuffer, position); 55 | intBuffer.rewind(); 56 | return intBuffer.getInt(); 57 | } 58 | 59 | @Override 60 | public long readLong(long position) throws IOException { 61 | ByteBuffer longBuffer = longBuffer(); 62 | channel.read(longBuffer, position); 63 | longBuffer.rewind(); 64 | return longBuffer.getLong(); 65 | } 66 | 67 | @Override 68 | public long size() throws IOException { 69 | return channel.size(); 70 | } 71 | 72 | @Override 73 | public void close() throws IOException { 74 | channel.close(); 75 | } 76 | 77 | private ByteBuffer intBuffer() { 78 | ByteBuffer buffer = primitiveBuffer.get(); 79 | buffer.rewind(); 80 | buffer.limit(Sizes.INT_SIZE); 81 | return buffer; 82 | } 83 | 84 | private ByteBuffer longBuffer() { 85 | ByteBuffer buffer = primitiveBuffer.get(); 86 | buffer.rewind(); 87 | buffer.limit(Sizes.LONG_SIZE); 88 | return buffer; 89 | } 90 | 91 | public static ImmutableFile open(Path path) throws IOException { 92 | FileChannel channel = FileChannel.open(path, StandardOpenOption.READ); 93 | return new ImmutableChannelFile(channel); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/io/ImmutableFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.io; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * An immutable file. 25 | */ 26 | public interface ImmutableFile extends Closeable { 27 | 28 | public long read(ByteBuffer bufferToRead, long position) throws IOException; 29 | 30 | public int readInt(long position) throws IOException; 31 | 32 | public long readLong(long position) throws IOException; 33 | 34 | public long size() throws IOException; 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/io/Throttle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.io; 18 | 19 | import com.google.common.util.concurrent.RateLimiter; 20 | 21 | /** 22 | * An I/O rate limiter that uses the Token Bucket algorithm to limit throughput. 23 | */ 24 | public class Throttle { 25 | 26 | public static Throttle MAX = new Throttle(Integer.MAX_VALUE); 27 | 28 | private final RateLimiter rateLimiter; 29 | 30 | public Throttle(long maxRatePerSecond) { 31 | this.rateLimiter = RateLimiter.create(maxRatePerSecond); 32 | } 33 | 34 | public void consume(int usage) { 35 | rateLimiter.acquire(usage); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/metrics/CacheHitGauge.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.metrics; 18 | 19 | import com.codahale.metrics.Meter; 20 | import com.codahale.metrics.RatioGauge; 21 | 22 | /** 23 | * A RatioGauge that represents a cache hit rate. 24 | */ 25 | public class CacheHitGauge extends RatioGauge { 26 | 27 | private final Meter hits = new Meter(); 28 | private final Meter misses = new Meter(); 29 | 30 | public void hit() { 31 | hits.mark(); 32 | } 33 | 34 | public void miss() { 35 | misses.mark(); 36 | } 37 | 38 | public void sample(boolean success) { 39 | if (success) { 40 | hit(); 41 | } else { 42 | miss(); 43 | } 44 | } 45 | 46 | @Override 47 | public Ratio getRatio() { 48 | return Ratio.of(hits.getFiveMinuteRate(), misses.getFiveMinuteRate() + hits.getFiveMinuteRate()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/BitSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap; 18 | 19 | import com.jordanwilliams.heftydb.util.Sizes; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * An immutable bit set that is backed by off-heap memory. 25 | */ 26 | public class BitSet implements Offheap { 27 | 28 | public static class Builder { 29 | 30 | private final MemoryPointer pointer; 31 | private final ByteBuffer directBuffer; 32 | private final int usableBytes; 33 | 34 | public Builder(long bitCount, int paddingBytes) { 35 | this.usableBytes = memoryOffset(bitCount) + Sizes.LONG_SIZE; 36 | this.pointer = MemoryAllocator.allocateAndZero(usableBytes + paddingBytes); 37 | this.directBuffer = pointer.directBuffer(); 38 | } 39 | 40 | public void set(long bitIndex, boolean value) { 41 | int offset = memoryOffset(bitIndex); 42 | 43 | if (value) { 44 | //Set 45 | directBuffer.putLong(offset, directBuffer.getLong(offset) | (1L << bitIndex)); 46 | } else { 47 | //Clear 48 | directBuffer.putLong(offset, directBuffer.getLong(offset) & ~(1L << bitIndex)); 49 | } 50 | } 51 | 52 | public BitSet build() { 53 | return new BitSet(pointer, usableBytes); 54 | } 55 | 56 | public int usableBytes() { 57 | return usableBytes; 58 | } 59 | 60 | public long bitCount() { 61 | return usableBytes * 8; 62 | } 63 | } 64 | 65 | private static final int ADDRESS_BITS_PER_WORD = 6; 66 | 67 | private final MemoryPointer pointer; 68 | private final ByteBuffer directBuffer; 69 | private final int usableBytes; 70 | 71 | public BitSet(MemoryPointer pointer, int usableBytes) { 72 | this.pointer = pointer; 73 | this.directBuffer = pointer.directBuffer(); 74 | this.usableBytes = usableBytes; 75 | } 76 | 77 | public boolean get(long index) { 78 | int offset = memoryOffset(index); 79 | long currentValue = directBuffer.getLong(offset); 80 | return (currentValue & (1L << index)) != 0; 81 | } 82 | 83 | public int usableBytes() { 84 | return usableBytes; 85 | } 86 | 87 | public long bitCount() { 88 | return usableBytes * 8; 89 | } 90 | 91 | @Override 92 | public MemoryPointer memory() { 93 | return pointer; 94 | } 95 | 96 | private static int memoryOffset(long bitIndex) { 97 | return (int) (bitIndex >> ADDRESS_BITS_PER_WORD) * Sizes.LONG_SIZE; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/JVMUnsafe.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap; 18 | 19 | import java.lang.reflect.Field; 20 | 21 | /** 22 | * Provides sane access to sun.misc.Unsafe 23 | */ 24 | public class JVMUnsafe { 25 | 26 | public static final sun.misc.Unsafe unsafe; 27 | 28 | static { 29 | try { 30 | Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); 31 | field.setAccessible(true); 32 | unsafe = (sun.misc.Unsafe) field.get(null); 33 | } catch (Exception e) { 34 | throw new AssertionError(e); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/MemoryPointer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap; 18 | 19 | import java.nio.ByteBuffer; 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | /** 23 | * Represents a reference counted pointer to a block of off-heap memory. Each MemoryPointer has a DirectByteBuffer 24 | * instance that can be used to safely access the memory the pointer points to. Reference counting operations can be 25 | * done safely from multiple concurrent threads. 26 | */ 27 | public class MemoryPointer { 28 | 29 | private final AtomicInteger retainCount = new AtomicInteger(1); 30 | private final int size; 31 | private final ByteBuffer directBuffer; 32 | 33 | private long address; 34 | 35 | MemoryPointer(long address, int size, ByteBuffer directBuffer) { 36 | this.address = address; 37 | this.size = size; 38 | this.directBuffer = directBuffer; 39 | directBuffer.rewind(); 40 | } 41 | 42 | public long address() { 43 | return address; 44 | } 45 | 46 | public ByteBuffer directBuffer() { 47 | return directBuffer; 48 | } 49 | 50 | public boolean isFree() { 51 | return address == 0; 52 | } 53 | 54 | public void free() { 55 | MemoryAllocator.deallocate(address, size); 56 | retainCount.set(0); 57 | address = 0; 58 | } 59 | 60 | public int size() { 61 | return directBuffer.limit(); 62 | } 63 | 64 | public boolean retain() { 65 | while (true) { 66 | int retainValue = retainCount.get(); 67 | 68 | if (retainValue <= 0) { 69 | return false; 70 | } 71 | 72 | if (retainCount.compareAndSet(retainValue, retainValue + 1)) { 73 | return true; 74 | } 75 | } 76 | } 77 | 78 | public void release() { 79 | if (retainCount.decrementAndGet() == 0) { 80 | free(); 81 | } 82 | } 83 | 84 | @Override 85 | public String toString() { 86 | return "MemoryPointer{" + 87 | "retainCount=" + retainCount + 88 | ", size=" + size + 89 | ", address=" + address + 90 | ", contents=" + toHexString(directBuffer) + 91 | '}'; 92 | } 93 | 94 | private static char[] hexArray = "0123456789ABCDEF".toCharArray(); 95 | 96 | private static String toHexString(ByteBuffer byteBuffer) { 97 | char[] hexChars = new char[byteBuffer.capacity() * 2]; 98 | for (int i = 0; i < byteBuffer.capacity(); i++) { 99 | int v = byteBuffer.get(i) & 0xFF; 100 | hexChars[i * 2] = hexArray[v >>> 4]; 101 | hexChars[i * 2 + 1] = hexArray[v & 0x0F]; 102 | } 103 | return new String(hexChars); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/Offheap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap; 18 | 19 | /** 20 | * Indicates that the implementing class is backed by off-heap memory. 21 | */ 22 | public interface Offheap { 23 | 24 | public MemoryPointer memory(); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/allocator/Allocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap.allocator; 18 | 19 | /** 20 | * An off-heap memory allocator. 21 | */ 22 | public interface Allocator { 23 | 24 | public long allocate(long bytes); 25 | 26 | public void deallocate(long address); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/offheap/allocator/UnsafeAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.offheap.allocator; 18 | 19 | 20 | import com.jordanwilliams.heftydb.offheap.JVMUnsafe; 21 | import sun.misc.Unsafe; 22 | 23 | /** 24 | * An off-heap memory allocator that uses sun.misc.Unsafe 25 | */ 26 | public class UnsafeAllocator implements Allocator { 27 | 28 | private static final Unsafe unsafe = JVMUnsafe.unsafe; 29 | 30 | @Override 31 | public long allocate(long size) { 32 | return unsafe.allocateMemory(size); 33 | } 34 | 35 | @Override 36 | public void deallocate(long address) { 37 | unsafe.freeMemory(address); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/read/CompactionTupleIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.read; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.util.CloseableIterator; 21 | 22 | import java.io.IOException; 23 | import java.util.LinkedList; 24 | import java.util.Queue; 25 | import java.util.SortedSet; 26 | import java.util.TreeSet; 27 | 28 | /** 29 | * An Iterator that filters a sorted stream of Tuples, and filters out all key versions older than a minimum snapshot 30 | * id, or passes along a key if there is only one version of it 31 | */ 32 | public class CompactionTupleIterator implements CloseableIterator { 33 | 34 | private final CloseableIterator tupleIterator; 35 | private final Queue nextTuples = new LinkedList(); 36 | private final SortedSet currentKeyTuples = new TreeSet(); 37 | private final long minSnapshotId; 38 | 39 | public CompactionTupleIterator(long minSnapshotId, CloseableIterator tupleIterator) { 40 | this.minSnapshotId = minSnapshotId; 41 | this.tupleIterator = tupleIterator; 42 | } 43 | 44 | @Override 45 | public boolean hasNext() { 46 | if (!nextTuples.isEmpty()) { 47 | return true; 48 | } 49 | 50 | if (!fetchNextTuples()){ 51 | return false; 52 | } 53 | 54 | return true; 55 | } 56 | 57 | @Override 58 | public Tuple next() { 59 | if (nextTuples.isEmpty()) { 60 | hasNext(); 61 | } 62 | 63 | return nextTuples.poll(); 64 | } 65 | 66 | @Override 67 | public void remove() { 68 | throw new UnsupportedOperationException(); 69 | } 70 | 71 | @Override 72 | public void close() throws IOException { 73 | tupleIterator.close(); 74 | } 75 | 76 | private boolean fetchNextTuples() { 77 | while (tupleIterator.hasNext()) { 78 | Tuple next = tupleIterator.next(); 79 | 80 | boolean nextKeyEqualCurrent = currentKeyTuples.isEmpty() || next.key().data().equals(currentKeyTuples 81 | .last().key().data()); 82 | 83 | if (nextKeyEqualCurrent) { 84 | currentKeyTuples.add(next); 85 | continue; 86 | } 87 | 88 | filterCurrentKeyTuples(); 89 | currentKeyTuples.clear(); 90 | currentKeyTuples.add(next); 91 | 92 | return true; 93 | } 94 | 95 | if (currentKeyTuples.isEmpty()) { 96 | return false; 97 | } 98 | 99 | filterCurrentKeyTuples(); 100 | currentKeyTuples.clear(); 101 | return true; 102 | } 103 | 104 | private void filterCurrentKeyTuples(){ 105 | int count = 0; 106 | 107 | for (Tuple tuple : currentKeyTuples){ 108 | if (tuple.key().snapshotId() >= minSnapshotId){ 109 | nextTuples.add(tuple); 110 | count++; 111 | } 112 | } 113 | 114 | if (count == 0){ 115 | nextTuples.add(currentKeyTuples.last()); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/read/LatestTupleIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.read; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.util.CloseableIterator; 21 | 22 | import java.io.IOException; 23 | import java.util.LinkedList; 24 | import java.util.Queue; 25 | import java.util.SortedSet; 26 | import java.util.TreeSet; 27 | 28 | /** 29 | * An Iterator that filters a sorted stream of Tuples, and returns only a single Tuple for each unique Key in the 30 | * stream that is the latest version of that particular Tuple according to the snapshot id associated with each key. 31 | */ 32 | public class LatestTupleIterator implements CloseableIterator { 33 | 34 | private final CloseableIterator tupleIterator; 35 | private final Queue nextTuple = new LinkedList(); 36 | private final long maxSnapshotId; 37 | private final SortedSet currentKeyTuples = new TreeSet(); 38 | 39 | public LatestTupleIterator(long maxSnapshotId, CloseableIterator tupleIterator) { 40 | this.maxSnapshotId = maxSnapshotId; 41 | this.tupleIterator = tupleIterator; 42 | } 43 | 44 | @Override 45 | public boolean hasNext() { 46 | if (!nextTuple.isEmpty()) { 47 | return true; 48 | } 49 | 50 | Tuple tuple = fetchNextTuple(); 51 | 52 | if (tuple == null) { 53 | return false; 54 | } 55 | 56 | nextTuple.add(tuple); 57 | 58 | return true; 59 | } 60 | 61 | @Override 62 | public Tuple next() { 63 | if (nextTuple.isEmpty()) { 64 | hasNext(); 65 | } 66 | 67 | return nextTuple.poll(); 68 | } 69 | 70 | @Override 71 | public void remove() { 72 | throw new UnsupportedOperationException(); 73 | } 74 | 75 | private Tuple fetchNextTuple() { 76 | while (tupleIterator.hasNext()) { 77 | Tuple next = tupleIterator.next(); 78 | 79 | if (next.key().snapshotId() > maxSnapshotId) { 80 | continue; 81 | } 82 | 83 | boolean nextKeyEqualCurrent = currentKeyTuples.isEmpty() || next.key().data().equals(currentKeyTuples 84 | .last().key().data()); 85 | 86 | if (nextKeyEqualCurrent) { 87 | currentKeyTuples.add(next); 88 | continue; 89 | } 90 | 91 | Tuple newest = currentKeyTuples.last(); 92 | currentKeyTuples.clear(); 93 | currentKeyTuples.add(next); 94 | return newest; 95 | } 96 | 97 | if (currentKeyTuples.isEmpty()) { 98 | return null; 99 | } 100 | 101 | Tuple newest = currentKeyTuples.last(); 102 | currentKeyTuples.clear(); 103 | return newest; 104 | } 105 | 106 | @Override 107 | public void close() throws IOException { 108 | tupleIterator.close(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/state/Caches.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.state; 18 | 19 | import com.jordanwilliams.heftydb.index.IndexBlock; 20 | import com.jordanwilliams.heftydb.table.file.TupleBlock; 21 | 22 | /** 23 | * Encapsulates the BlockCaches used in a database. 24 | */ 25 | public class Caches { 26 | 27 | private final TupleBlock.Cache recordBlockCache; 28 | private final IndexBlock.Cache indexBlockCache; 29 | 30 | public Caches(TupleBlock.Cache recordBlockCache, IndexBlock.Cache indexBlockCache) { 31 | this.recordBlockCache = recordBlockCache; 32 | this.indexBlockCache = indexBlockCache; 33 | } 34 | 35 | public TupleBlock.Cache recordBlockCache() { 36 | return recordBlockCache; 37 | } 38 | 39 | public IndexBlock.Cache indexBlockCache() { 40 | return indexBlockCache; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/state/Paths.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.state; 18 | 19 | import java.io.IOException; 20 | import java.nio.file.DirectoryStream; 21 | import java.nio.file.Files; 22 | import java.nio.file.Path; 23 | import java.util.SortedSet; 24 | import java.util.TreeSet; 25 | 26 | /** 27 | * Generates various file paths for all the files in a database. 28 | */ 29 | public class Paths { 30 | 31 | private static final String TABLE_EXT = ".table"; 32 | private static final String LOG_EXT = ".log"; 33 | private static final String INDEX_EXT = ".index"; 34 | private static final String FILTER_EXT = ".filter"; 35 | private static final String TEMP_EXT = ".temp"; 36 | 37 | private final Path logDirectory; 38 | private final Path tableDirectory; 39 | 40 | public Paths(Path tableDirectory, Path logDirectory) { 41 | this.tableDirectory = tableDirectory; 42 | this.logDirectory = logDirectory; 43 | } 44 | 45 | public Path tablePath(long tableId) { 46 | return tableDirectory.resolve(tableId + TABLE_EXT); 47 | } 48 | 49 | public Path indexPath(long tableId) { 50 | return tableDirectory.resolve(tableId + INDEX_EXT); 51 | } 52 | 53 | public Path filterPath(long tableId) { 54 | return tableDirectory.resolve(tableId + FILTER_EXT); 55 | } 56 | 57 | public Path logPath(long tableId) { 58 | return tableDirectory.resolve(tableId + LOG_EXT); 59 | } 60 | 61 | public Path tempPath(long tableId) { 62 | return tableDirectory.resolve(tableId + TEMP_EXT); 63 | } 64 | 65 | public SortedSet tableFileIds() throws IOException { 66 | return fileIds(tableFilePaths()); 67 | } 68 | 69 | public SortedSet tempTableFileIds() throws IOException { 70 | return fileIds(tempFilePaths()); 71 | } 72 | 73 | public SortedSet logFileIds() throws IOException { 74 | return fileIds(logFilePaths()); 75 | } 76 | 77 | public SortedSet tableFilePaths() throws IOException { 78 | return filePaths(tableDirectory, TABLE_EXT); 79 | } 80 | 81 | public SortedSet logFilePaths() throws IOException { 82 | return filePaths(tableDirectory, LOG_EXT); 83 | } 84 | 85 | public SortedSet tempFilePaths() throws IOException { 86 | return filePaths(tableDirectory, TEMP_EXT); 87 | } 88 | 89 | private SortedSet filePaths(Path directory, String extension) throws IOException { 90 | SortedSet sortedFilePaths = new TreeSet(); 91 | DirectoryStream filePaths = Files.newDirectoryStream(directory, "*" + extension); 92 | 93 | for (Path filePath : filePaths) { 94 | sortedFilePaths.add(filePath); 95 | } 96 | 97 | filePaths.close(); 98 | 99 | return sortedFilePaths; 100 | } 101 | 102 | private SortedSet fileIds(SortedSet paths) { 103 | SortedSet sortedFileIds = new TreeSet(); 104 | 105 | for (Path path : paths) { 106 | sortedFileIds.add(tableId(path)); 107 | } 108 | 109 | return sortedFileIds; 110 | } 111 | 112 | private static long tableId(Path path) { 113 | String fileName = path.getFileName().toString(); 114 | String id = fileName.split("\\.")[0]; 115 | return Long.parseLong(id); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/state/Snapshots.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.state; 18 | 19 | import java.util.SortedSet; 20 | import java.util.TreeSet; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | /** 24 | * Keeps track of Snapshot ids in a database. 25 | */ 26 | public class Snapshots { 27 | 28 | private final AtomicLong currentSnapshotId = new AtomicLong(); 29 | private final SortedSet retainedSnapshots = new TreeSet(); 30 | 31 | public Snapshots(long startingSnapshotId) { 32 | this.currentSnapshotId.set(startingSnapshotId); 33 | retainedSnapshots.add(Long.MAX_VALUE); 34 | } 35 | 36 | public long nextId() { 37 | return currentSnapshotId.incrementAndGet(); 38 | } 39 | 40 | public long currentId() { 41 | return currentSnapshotId.get(); 42 | } 43 | 44 | public long minimumRetainedId(){ 45 | return retainedSnapshots.first(); 46 | } 47 | 48 | public synchronized void retain(long snapshotId){ 49 | retainedSnapshots.add(snapshotId); 50 | } 51 | 52 | public synchronized void release(long snapshotId){ 53 | retainedSnapshots.remove(snapshotId); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/MutableTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | 21 | /** 22 | * Represents a mutable Table. 23 | */ 24 | public interface MutableTable extends Table { 25 | 26 | public void put(Tuple tuple); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/Table.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.util.CloseableIterator; 22 | 23 | /** 24 | * Represents an immutable collection of sorted record in a database. 25 | */ 26 | public interface Table extends Iterable, Comparable
{ 27 | 28 | public long id(); 29 | 30 | public boolean mightContain(Key key); 31 | 32 | public Tuple get(Key key); 33 | 34 | public CloseableIterator ascendingIterator(long snapshotId); 35 | 36 | public CloseableIterator descendingIterator(long snapshotId); 37 | 38 | public CloseableIterator ascendingIterator(Key key, long snapshotId); 39 | 40 | public CloseableIterator descendingIterator(Key key, long snapshotId); 41 | 42 | public long tupleCount(); 43 | 44 | public long size(); 45 | 46 | public int level(); 47 | 48 | public long maxSnapshotId(); 49 | 50 | public void close(); 51 | 52 | public boolean isPersistent(); 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/file/TableBloomFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.io.ImmutableChannelFile; 21 | import com.jordanwilliams.heftydb.io.ImmutableFile; 22 | import com.jordanwilliams.heftydb.offheap.BloomFilter; 23 | import com.jordanwilliams.heftydb.offheap.MemoryAllocator; 24 | import com.jordanwilliams.heftydb.offheap.MemoryPointer; 25 | import com.jordanwilliams.heftydb.offheap.Offheap; 26 | import com.jordanwilliams.heftydb.state.Paths; 27 | 28 | import java.io.IOException; 29 | import java.nio.ByteBuffer; 30 | 31 | /** 32 | * Reads an immutable BloomFilter from a file. 33 | */ 34 | public class TableBloomFilter implements Offheap { 35 | 36 | private final BloomFilter bloomFilter; 37 | 38 | private TableBloomFilter(BloomFilter bloomFilter) throws IOException { 39 | this.bloomFilter = bloomFilter; 40 | } 41 | 42 | public boolean mightContain(Key key) { 43 | return bloomFilter.mightContain(key); 44 | } 45 | 46 | public void close() { 47 | bloomFilter.memory().release(); 48 | } 49 | 50 | @Override 51 | public MemoryPointer memory() { 52 | return bloomFilter.memory(); 53 | } 54 | 55 | public static TableBloomFilter read(long tableId, Paths paths) throws IOException { 56 | ImmutableFile filterFile = ImmutableChannelFile.open(paths.filterPath(tableId)); 57 | MemoryPointer filterPointer = MemoryAllocator.allocate((int) filterFile.size()); 58 | ByteBuffer filterBuffer = filterPointer.directBuffer(); 59 | filterFile.read(filterBuffer, 0); 60 | filterFile.close(); 61 | return new TableBloomFilter(new BloomFilter(filterPointer)); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/file/TableBloomFilterWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.io.AppendChannelFile; 21 | import com.jordanwilliams.heftydb.io.AppendFile; 22 | import com.jordanwilliams.heftydb.offheap.BloomFilter; 23 | import com.jordanwilliams.heftydb.state.Paths; 24 | 25 | import java.io.IOException; 26 | import java.nio.ByteBuffer; 27 | 28 | /** 29 | * Writes a BloomFilter out to a file. 30 | */ 31 | public class TableBloomFilterWriter { 32 | 33 | private static final double FALSE_POSITIVE_PROBABILITY = 0.01; 34 | 35 | private final BloomFilter.Builder filterBuilder; 36 | private final AppendFile filterFile; 37 | 38 | private TableBloomFilterWriter(AppendFile filterFile, long approxRecordCount) { 39 | this.filterBuilder = new BloomFilter.Builder(approxRecordCount, FALSE_POSITIVE_PROBABILITY); 40 | this.filterFile = filterFile; 41 | } 42 | 43 | public void write(Key key) throws IOException { 44 | filterBuilder.put(key); 45 | } 46 | 47 | public void finish() throws IOException { 48 | BloomFilter filter = filterBuilder.build(); 49 | ByteBuffer filterBuffer = filter.memory().directBuffer(); 50 | filterFile.append(filterBuffer); 51 | filterFile.close(); 52 | filter.memory().release(); 53 | } 54 | 55 | public static TableBloomFilterWriter open(long tableId, Paths paths, long approxRecordCount) throws IOException { 56 | AppendFile filterFile = AppendChannelFile.open(paths.filterPath(tableId)); 57 | return new TableBloomFilterWriter(filterFile, approxRecordCount); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/file/TableTrailer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.io.ImmutableFile; 21 | 22 | import java.io.IOException; 23 | import java.nio.ByteBuffer; 24 | 25 | /** 26 | * Encapsulates meta data stored at the end of a Table file. 27 | */ 28 | public class TableTrailer { 29 | 30 | public static final int SIZE = 28; 31 | 32 | public static class Builder { 33 | 34 | private final long tableId; 35 | private final int level; 36 | private long recordCount; 37 | private long maxSnapshotId; 38 | 39 | public Builder(long tableId, int level) { 40 | this.tableId = tableId; 41 | this.level = level; 42 | } 43 | 44 | public void put(Tuple tuple) { 45 | maxSnapshotId = Math.max(tuple.key().snapshotId(), maxSnapshotId); 46 | recordCount++; 47 | } 48 | 49 | public TableTrailer build() { 50 | return new TableTrailer(serialize()); 51 | } 52 | 53 | private ByteBuffer serialize() { 54 | ByteBuffer trailerBuffer = ByteBuffer.allocate(SIZE); 55 | trailerBuffer.putLong(tableId); 56 | trailerBuffer.putInt(level); 57 | trailerBuffer.putLong(recordCount); 58 | trailerBuffer.putLong(maxSnapshotId); 59 | trailerBuffer.rewind(); 60 | return trailerBuffer; 61 | } 62 | } 63 | 64 | private final ByteBuffer buffer; 65 | private final long tableId; 66 | private final int level; 67 | private final long recordCount; 68 | private final long maxSnapshotId; 69 | 70 | public TableTrailer(ByteBuffer buffer) { 71 | this.tableId = buffer.getLong(); 72 | this.level = buffer.getInt(); 73 | this.recordCount = buffer.getLong(); 74 | this.maxSnapshotId = buffer.getLong(); 75 | buffer.rewind(); 76 | this.buffer = buffer; 77 | } 78 | 79 | public long tableId() { 80 | return tableId; 81 | } 82 | 83 | public long maxSnapshotId() { 84 | return maxSnapshotId; 85 | } 86 | 87 | public long recordCount() { 88 | return recordCount; 89 | } 90 | 91 | public int level() { 92 | return level; 93 | } 94 | 95 | public ByteBuffer buffer() { 96 | return buffer; 97 | } 98 | 99 | public static TableTrailer read(ImmutableFile tableFile) throws IOException { 100 | ByteBuffer trailerBuffer = ByteBuffer.allocate(SIZE); 101 | tableFile.read(trailerBuffer, tableFile.size() - SIZE); 102 | trailerBuffer.rewind(); 103 | return new TableTrailer(trailerBuffer); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/memory/MemoryTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.memory; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.table.MutableTable; 22 | import com.jordanwilliams.heftydb.table.Table; 23 | import com.jordanwilliams.heftydb.util.CloseableIterator; 24 | 25 | import java.util.Iterator; 26 | import java.util.concurrent.atomic.AtomicInteger; 27 | import java.util.concurrent.atomic.AtomicLong; 28 | 29 | /** 30 | * Provides a MutableTable that is kept in Memory. Records are kept in sorted order as they are inserted. 31 | */ 32 | public class MemoryTable implements MutableTable { 33 | 34 | private final long id; 35 | private final SortedTupleMap records = new SkipListTupleMap(); 36 | private final AtomicLong maxSnapshotId = new AtomicLong(); 37 | private final AtomicInteger recordCount = new AtomicInteger(); 38 | private final AtomicInteger size = new AtomicInteger(); 39 | 40 | public MemoryTable(long id) { 41 | this.id = id; 42 | } 43 | 44 | @Override 45 | public void put(Tuple tuple) { 46 | records.put(tuple.key(), tuple.value()); 47 | recordCount.incrementAndGet(); 48 | size.addAndGet(tuple.size()); 49 | maxSnapshotId.set(tuple.key().snapshotId()); 50 | } 51 | 52 | @Override 53 | public long id() { 54 | return id; 55 | } 56 | 57 | @Override 58 | public boolean mightContain(Key key) { 59 | return get(key) != null; 60 | } 61 | 62 | @Override 63 | public Tuple get(Key key) { 64 | return records.get(key); 65 | } 66 | 67 | @Override 68 | public CloseableIterator ascendingIterator(long snapshotId) { 69 | return records.ascendingIterator(snapshotId); 70 | } 71 | 72 | @Override 73 | public CloseableIterator descendingIterator(long snapshotId) { 74 | return records.descendingIterator(snapshotId); 75 | } 76 | 77 | @Override 78 | public CloseableIterator ascendingIterator(Key key, long snapshotId) { 79 | return records.ascendingIterator(key, snapshotId); 80 | } 81 | 82 | @Override 83 | public CloseableIterator descendingIterator(Key key, long snapshotId) { 84 | return records.descendingIterator(key, snapshotId); 85 | } 86 | 87 | @Override 88 | public long tupleCount() { 89 | return recordCount.get(); 90 | } 91 | 92 | @Override 93 | public long size() { 94 | return size.get(); 95 | } 96 | 97 | @Override 98 | public int level() { 99 | return 0; 100 | } 101 | 102 | @Override 103 | public long maxSnapshotId() { 104 | return maxSnapshotId.get(); 105 | } 106 | 107 | @Override 108 | public void close() { 109 | } 110 | 111 | @Override 112 | public boolean isPersistent() { 113 | return false; 114 | } 115 | 116 | @Override 117 | public Iterator iterator() { 118 | return records.iterator(); 119 | } 120 | 121 | @Override 122 | public int compareTo(Table o) { 123 | return Long.compare(id, o.id()); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/memory/SkipListTupleMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.memory; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | import com.jordanwilliams.heftydb.read.LatestTupleIterator; 23 | import com.jordanwilliams.heftydb.util.CloseableIterator; 24 | 25 | import java.util.Iterator; 26 | import java.util.Map; 27 | import java.util.concurrent.ConcurrentNavigableMap; 28 | import java.util.concurrent.ConcurrentSkipListMap; 29 | 30 | /** 31 | * A SortedTupleMap backed by a ConcurrentSkipListMap. This class is safe to use from multiple concurrent threads. 32 | */ 33 | public class SkipListTupleMap implements SortedTupleMap { 34 | 35 | private final ConcurrentNavigableMap tuples = new ConcurrentSkipListMap(); 36 | 37 | @Override 38 | public void put(Key key, Value value) { 39 | tuples.put(key, new Tuple(key, value)); 40 | } 41 | 42 | @Override 43 | public Tuple get(Key key) { 44 | Map.Entry closestEntry = tuples.floorEntry(key); 45 | 46 | if (closestEntry == null) { 47 | return null; 48 | } 49 | 50 | return closestEntry.getKey().data().equals(key.data()) ? closestEntry.getValue() : null; 51 | } 52 | 53 | @Override 54 | public CloseableIterator ascendingIterator(long snapshotId) { 55 | return new LatestTupleIterator(snapshotId, new CloseableIterator.Wrapper(tuples.values().iterator())); 56 | } 57 | 58 | @Override 59 | public CloseableIterator descendingIterator(long snapshotId) { 60 | return new LatestTupleIterator(snapshotId, new CloseableIterator.Wrapper(tuples.descendingMap().values 61 | ().iterator())); 62 | } 63 | 64 | @Override 65 | public CloseableIterator ascendingIterator(Key key, long snapshotId) { 66 | return new LatestTupleIterator(snapshotId, new CloseableIterator.Wrapper(tuples.tailMap(key, 67 | true).values().iterator())); 68 | } 69 | 70 | @Override 71 | public CloseableIterator descendingIterator(Key key, long snapshotId) { 72 | return new LatestTupleIterator(snapshotId, new CloseableIterator.Wrapper(tuples.headMap(key, 73 | true).descendingMap().values().iterator())); 74 | } 75 | 76 | @Override 77 | public Iterator iterator() { 78 | return tuples.values().iterator(); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/table/memory/SortedTupleMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.table.memory; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | import com.jordanwilliams.heftydb.util.CloseableIterator; 23 | 24 | /** 25 | * Provides the underlying sorted storage for a MemoryTable. 26 | */ 27 | public interface SortedTupleMap extends Iterable { 28 | 29 | public void put(Key key, Value value); 30 | 31 | public Tuple get(Key key); 32 | 33 | public CloseableIterator ascendingIterator(long snapshotId); 34 | 35 | public CloseableIterator descendingIterator(long snapshotId); 36 | 37 | public CloseableIterator ascendingIterator(Key key, long snapshotId); 38 | 39 | public CloseableIterator descendingIterator(Key key, long snapshotId); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/ByteBuffers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | import java.nio.ByteBuffer; 20 | import java.nio.charset.Charset; 21 | 22 | /** 23 | * Some utilities for working with ByteBuffers. 24 | */ 25 | public class ByteBuffers { 26 | 27 | public static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); 28 | 29 | public static ByteBuffer fromString(String string) { 30 | return ByteBuffer.wrap(string.getBytes(Charset.defaultCharset())); 31 | } 32 | 33 | public static String toString(ByteBuffer byteBuffer) { 34 | byte[] bytes = new byte[byteBuffer.capacity()]; 35 | 36 | for (int i = 0; i < byteBuffer.capacity(); i++) { 37 | bytes[i] = byteBuffer.get(i); 38 | } 39 | 40 | return new String(bytes); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/CloseableIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.util.Iterator; 22 | 23 | /** 24 | * An iterator that needs to be closed when it is no longer used so that off-heap memory or file handles can be 25 | * properly cleaned up. 26 | * @param 27 | */ 28 | public interface CloseableIterator extends Iterator, Closeable { 29 | 30 | public static class Wrapper implements CloseableIterator { 31 | 32 | private final Iterator delegate; 33 | 34 | public Wrapper(Iterator delegate) { 35 | this.delegate = delegate; 36 | } 37 | 38 | @Override 39 | public void close() throws IOException { 40 | 41 | } 42 | 43 | @Override 44 | public boolean hasNext() { 45 | return delegate.hasNext(); 46 | } 47 | 48 | @Override 49 | public T next() { 50 | return delegate.next(); 51 | } 52 | 53 | @Override 54 | public void remove() { 55 | delegate.remove(); 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/PeekableIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | /** 20 | * An iterator that allows peeking at the current value. 21 | * @param 22 | */ 23 | public interface PeekableIterator { 24 | 25 | public T current(); 26 | 27 | public boolean advance(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/Serializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | /** 23 | * A generic serializer that writes objects to ByteBuffers. 24 | * @param 25 | */ 26 | public interface Serializer { 27 | 28 | public int size(V item); 29 | 30 | public void serialize(V item, ByteBuffer buffer); 31 | 32 | public V deserialize(ByteBuffer buffer); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/Sizes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | /** 20 | * JVM primitive sizes in bytes. 21 | */ 22 | public class Sizes { 23 | 24 | public static int LONG_SIZE = 8; 25 | 26 | public static int INT_SIZE = 4; 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/util/XORShiftRandom.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.util; 18 | 19 | /** 20 | * A simple xor shift random number generator. Generates a predictable stream of pseudo-random numbers from a 21 | * starting seed. 22 | */ 23 | public class XORShiftRandom { 24 | 25 | private long last; 26 | 27 | public XORShiftRandom(long seed) { 28 | this.last = seed; 29 | } 30 | 31 | public int nextInt(int max) { 32 | last ^= (last << 21); 33 | last ^= (last >>> 35); 34 | last ^= (last << 4); 35 | int out = (int) last % max; 36 | return (out < 0) ? -out : out; 37 | } 38 | 39 | public int nextInt() { 40 | last ^= (last << 21); 41 | last ^= (last >>> 35); 42 | last ^= (last << 4); 43 | int out = (int) last; 44 | return (out < 0) ? -out : out; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/jordanwilliams/heftydb/write/CommitLogWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.write; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.io.AppendChannelFile; 21 | import com.jordanwilliams.heftydb.io.AppendFile; 22 | import com.jordanwilliams.heftydb.state.Paths; 23 | import com.jordanwilliams.heftydb.util.XORShiftRandom; 24 | 25 | import java.io.Closeable; 26 | import java.io.IOException; 27 | 28 | /** 29 | * Writes a CommitLog file. Each write can optionally be fsynced if required. 30 | */ 31 | public class CommitLogWriter implements Closeable { 32 | 33 | private final long tableId; 34 | private final XORShiftRandom pseudoRandom; 35 | private final AppendFile logFile; 36 | 37 | private CommitLogWriter(long tableId, AppendFile logFile) throws IOException { 38 | long seed = System.nanoTime(); 39 | this.tableId = tableId; 40 | this.pseudoRandom = new XORShiftRandom(seed); 41 | this.logFile = logFile; 42 | 43 | logFile.appendLong(seed); 44 | } 45 | 46 | public void append(Tuple tuple, boolean fsync) throws IOException { 47 | int serializedSize = Tuple.SERIALIZER.size(tuple); 48 | 49 | //Serialize in place to avoid an extra copy 50 | tuple.rewind(); 51 | logFile.appendInt(serializedSize); 52 | 53 | logFile.appendInt(tuple.key().size()); 54 | logFile.append(tuple.key().data()); 55 | logFile.appendLong(tuple.key().snapshotId()); 56 | 57 | logFile.appendInt(tuple.value().size()); 58 | logFile.append(tuple.value().data()); 59 | logFile.appendInt(pseudoRandom.nextInt()); 60 | tuple.rewind(); 61 | 62 | if (fsync) { 63 | logFile.sync(); 64 | } 65 | } 66 | 67 | public long tableId() { 68 | return tableId; 69 | } 70 | 71 | @Override 72 | public void close() throws IOException { 73 | logFile.close(); 74 | } 75 | 76 | public static CommitLogWriter open(long tableId, Paths paths) throws IOException { 77 | AppendFile logFile = AppendChannelFile.open(paths.logPath(tableId)); 78 | return new CommitLogWriter(tableId, logFile); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${user.dir}/logs/heftydb.log 5 | 6 | ${user.dir}/logs/heftydb.%d{yyyy-MM-dd}.log 7 | 30 8 | 9 | 10 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/base/FileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.base; 18 | 19 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 20 | import org.junit.After; 21 | import org.junit.AfterClass; 22 | import org.junit.Before; 23 | import org.junit.BeforeClass; 24 | 25 | import java.io.IOException; 26 | 27 | public class FileTest { 28 | 29 | @BeforeClass 30 | public static void beforeClass() throws IOException { 31 | TestFileHelper.createTestDirectory(); 32 | TestFileHelper.cleanUpTestFiles(); 33 | } 34 | 35 | @AfterClass 36 | public static void afterClass() throws IOException { 37 | TestFileHelper.cleanUpTestFiles(); 38 | } 39 | 40 | @Before 41 | public void beforeTest() throws IOException { 42 | TestFileHelper.createTestDirectory(); 43 | } 44 | 45 | @After 46 | public void afterTest() throws IOException { 47 | TestFileHelper.cleanUpTestFiles(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/base/ParameterizedIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.base; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.DB; 22 | import com.jordanwilliams.heftydb.db.HeftyDB; 23 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 24 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 25 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 26 | import org.junit.After; 27 | import org.junit.AfterClass; 28 | import org.junit.Before; 29 | import org.junit.BeforeClass; 30 | import org.junit.runner.RunWith; 31 | import org.junit.runners.Parameterized; 32 | 33 | import java.io.IOException; 34 | import java.util.ArrayList; 35 | import java.util.Collection; 36 | import java.util.List; 37 | import java.util.Random; 38 | 39 | @RunWith(Parameterized.class) 40 | public abstract class ParameterizedIntegrationTest { 41 | 42 | @BeforeClass 43 | public static void beforeClass() throws IOException { 44 | TestFileHelper.createTestDirectory(); 45 | TestFileHelper.cleanUpTestFiles(); 46 | } 47 | 48 | @AfterClass 49 | public static void afterClass() throws IOException { 50 | TestFileHelper.cleanUpTestFiles(); 51 | } 52 | 53 | @Before 54 | public void beforeTest() throws IOException { 55 | TestFileHelper.createTestDirectory(); 56 | } 57 | 58 | @After 59 | public void afterTest() throws IOException { 60 | TestFileHelper.cleanUpTestFiles(); 61 | } 62 | 63 | @Parameterized.Parameters 64 | public static Collection generateTestData() throws Exception { 65 | TupleGenerator tupleGenerator = new TupleGenerator(); 66 | final Random random = new Random(System.nanoTime()); 67 | List testParams = new ArrayList(); 68 | 69 | for (int i = 0; i < 100; i++) { 70 | Object[] params = new Object[2]; 71 | 72 | Config config = ConfigGenerator.testConfig(); 73 | List tuples = tupleGenerator.testRecords(1, 1000, 20, new TupleGenerator.Function() { 74 | @Override 75 | public Integer apply() { 76 | return random.nextInt(255) + 1; 77 | } 78 | }, new TupleGenerator.Function() { 79 | @Override 80 | public Integer apply() { 81 | return random.nextInt(255) + 1; 82 | } 83 | } 84 | ); 85 | params[0] = tuples; 86 | params[1] = config; 87 | 88 | testParams.add(params); 89 | } 90 | 91 | return testParams; 92 | } 93 | 94 | protected DB db; 95 | protected final List tuples; 96 | protected final Config config; 97 | 98 | public ParameterizedIntegrationTest(List tuples, Config config) throws IOException { 99 | this.db = HeftyDB.open(config); 100 | this.tuples = tuples; 101 | this.config = config; 102 | } 103 | 104 | protected void writeRecords() throws IOException { 105 | for (Tuple tuple : tuples) { 106 | db.put(tuple.key().data(), tuple.value().data()); 107 | } 108 | 109 | db.close(); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/base/ParameterizedTupleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.base; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 21 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 22 | import org.junit.After; 23 | import org.junit.AfterClass; 24 | import org.junit.Before; 25 | import org.junit.BeforeClass; 26 | import org.junit.runner.RunWith; 27 | import org.junit.runners.Parameterized; 28 | 29 | import java.io.IOException; 30 | import java.util.ArrayList; 31 | import java.util.Collection; 32 | import java.util.List; 33 | import java.util.Random; 34 | 35 | @RunWith(Parameterized.class) 36 | public abstract class ParameterizedTupleTest { 37 | 38 | @BeforeClass 39 | public static void beforeClass() throws IOException { 40 | TestFileHelper.createTestDirectory(); 41 | TestFileHelper.cleanUpTestFiles(); 42 | } 43 | 44 | @AfterClass 45 | public static void afterClass() throws IOException { 46 | TestFileHelper.cleanUpTestFiles(); 47 | } 48 | 49 | @Before 50 | public void beforeTest() throws IOException { 51 | tupleGenerator = new TupleGenerator(); 52 | TestFileHelper.createTestDirectory(); 53 | } 54 | 55 | @After 56 | public void afterTest() throws IOException { 57 | TestFileHelper.cleanUpTestFiles(); 58 | } 59 | 60 | @Parameterized.Parameters 61 | public static Collection generateTestRecords() { 62 | TupleGenerator tupleGenerator = new TupleGenerator(); 63 | List testParams = new ArrayList(); 64 | final Random random = new Random(System.nanoTime()); 65 | 66 | for (int i = 0; i < 100; i++) { 67 | Object[] params = new Object[1]; 68 | 69 | List tuples = tupleGenerator.testRecords(1, 1000, i, new TupleGenerator.Function() { 70 | @Override 71 | public Integer apply() { 72 | return random.nextInt(255) + 1; 73 | } 74 | }, new TupleGenerator.Function() { 75 | @Override 76 | public Integer apply() { 77 | return random.nextInt(255); 78 | } 79 | } 80 | ); 81 | params[0] = tuples; 82 | 83 | testParams.add(params); 84 | } 85 | 86 | return testParams; 87 | } 88 | 89 | protected final List tuples; 90 | protected TupleGenerator tupleGenerator; 91 | 92 | public ParameterizedTupleTest(List testTuples) { 93 | this.tuples = testTuples; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/base/TupleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.base; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 21 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 22 | import org.junit.After; 23 | import org.junit.AfterClass; 24 | import org.junit.Before; 25 | import org.junit.BeforeClass; 26 | 27 | import java.io.IOException; 28 | import java.util.List; 29 | import java.util.Random; 30 | 31 | public abstract class TupleTest { 32 | 33 | @BeforeClass 34 | public static void beforeClass() throws IOException { 35 | TestFileHelper.createTestDirectory(); 36 | TestFileHelper.cleanUpTestFiles(); 37 | } 38 | 39 | @AfterClass 40 | public static void afterClass() throws IOException { 41 | TestFileHelper.cleanUpTestFiles(); 42 | } 43 | 44 | @Before 45 | public void beforeTest() throws IOException { 46 | tupleGenerator = new TupleGenerator(); 47 | this.tuples = tupleGenerator.testRecords(1, 100, 20, random.nextInt(100) + 1, 100); 48 | TestFileHelper.createTestDirectory(); 49 | } 50 | 51 | @After 52 | public void afterTest() throws IOException { 53 | TestFileHelper.cleanUpTestFiles(); 54 | } 55 | 56 | protected final Random random = new Random(System.nanoTime()); 57 | protected TupleGenerator tupleGenerator; 58 | protected List tuples; 59 | 60 | protected List generateMoreTestRecords(int startingSnapshotId) { 61 | return tupleGenerator.testRecords(startingSnapshotId, 100, 20, random.nextInt(100) + 1, 100); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/generator/ConfigGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.generator; 18 | 19 | import com.jordanwilliams.heftydb.compact.CompactionStrategies; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.DBState; 22 | import com.jordanwilliams.heftydb.index.IndexBlock; 23 | import com.jordanwilliams.heftydb.metrics.Metrics; 24 | import com.jordanwilliams.heftydb.state.Caches; 25 | import com.jordanwilliams.heftydb.state.Paths; 26 | import com.jordanwilliams.heftydb.table.Table; 27 | import com.jordanwilliams.heftydb.table.file.TupleBlock; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | 30 | import java.util.Collections; 31 | 32 | public class ConfigGenerator { 33 | 34 | public static Paths testPaths() { 35 | return new Paths(TestFileHelper.TEMP_PATH, TestFileHelper.TEMP_PATH); 36 | } 37 | 38 | public static Caches testCaches() { 39 | return new Caches(new TupleBlock.Cache(32768000, new Metrics(testConfig())), new IndexBlock.Cache(16384000, 40 | new Metrics(testConfig()))); 41 | } 42 | 43 | public static Config defaultConfig() { 44 | return new Config.Builder().tableDirectory(TestFileHelper.TEMP_PATH).build(); 45 | } 46 | 47 | public static Config enduranceConfig() { 48 | return new Config.Builder().tableDirectory(TestFileHelper.TEMP_PATH).printMetrics(true).indexCacheSize 49 | (128000000).tableCacheSize(512000000).memoryTableSize(8192000).maxWriteRate(32768000).build(); 50 | } 51 | 52 | public static Config performanceConfig() { 53 | return new Config.Builder().tableDirectory(TestFileHelper.TEMP_PATH).indexCacheSize(128000000).tableCacheSize 54 | (512000000).tableBlockSize(1024).memoryTableSize(8192000).maxCompactionRate(32768000).build(); 55 | } 56 | 57 | public static Config testConfig() { 58 | Config.Builder builder = new Config.Builder(); 59 | 60 | return builder.compactionStrategy(CompactionStrategies.NULL_COMPACTION_STRATEGY).memoryTableSize(16384) 61 | .tableBlockSize(4096).indexBlockSize(4096).tableCacheSize(1024000).indexCacheSize(1024000) 62 | .tableDirectory(TestFileHelper.TEMP_PATH).build(); 63 | } 64 | 65 | public static DBState perfState() { 66 | return new DBState(Collections.
emptyList(), defaultConfig(), testPaths(), testCaches(), 1); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/generator/KeyValueGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.generator; 18 | 19 | import org.apache.commons.lang3.RandomStringUtils; 20 | 21 | import java.nio.ByteBuffer; 22 | import java.nio.charset.Charset; 23 | import java.util.ArrayList; 24 | import java.util.HashMap; 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.Random; 28 | 29 | public class KeyValueGenerator { 30 | 31 | private static final int MAX_CACHED_KEYS = 1024; 32 | 33 | private final Random rand = new Random(System.nanoTime()); 34 | private final List testKeys = new ArrayList(); 35 | private final Map testValues = new HashMap(); 36 | 37 | public synchronized ByteBuffer testKey(int size, int reuseWeight) { 38 | int next = rand.nextInt(100); 39 | 40 | if (next <= reuseWeight && reuseWeight != 0 && testKeys.size() > 0) { 41 | ByteBuffer existing = testKeys.get(rand.nextInt(testKeys.size())); 42 | ByteBuffer reusedKey = existing.duplicate(); 43 | reusedKey.rewind(); 44 | return reusedKey; 45 | } else { 46 | ByteBuffer random = randomKey(size); 47 | 48 | if (testKeys.size() < MAX_CACHED_KEYS) { 49 | testKeys.add(random); 50 | } 51 | 52 | return random.duplicate(); 53 | } 54 | } 55 | 56 | public ByteBuffer testKey(int reuseWeight) { 57 | return testKey(16, reuseWeight); 58 | } 59 | 60 | public synchronized ByteBuffer testValue(int size) { 61 | if (!testValues.containsKey(size)) { 62 | testValues.put(size, randomValue(size)); 63 | } 64 | 65 | ByteBuffer testValue = testValues.get(size).duplicate(); 66 | testValue.rewind(); 67 | return testValue; 68 | } 69 | 70 | private ByteBuffer randomKey(int size) { 71 | return ByteBuffer.wrap(RandomStringUtils.randomAlphanumeric(size).getBytes()); 72 | } 73 | 74 | private ByteBuffer randomValue(int size) { 75 | String randomString = RandomStringUtils.randomAlphanumeric(size); 76 | return ByteBuffer.wrap(randomString.getBytes(Charset.defaultCharset())); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/generator/TupleGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.generator; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | 23 | import java.nio.ByteBuffer; 24 | import java.util.ArrayList; 25 | import java.util.Collections; 26 | import java.util.List; 27 | import java.util.SortedMap; 28 | import java.util.TreeMap; 29 | 30 | public class TupleGenerator { 31 | 32 | public interface Function { 33 | public T apply(); 34 | } 35 | 36 | private final KeyValueGenerator testDataGenerator = new KeyValueGenerator(); 37 | 38 | public List testRecords(int startingSnapshotId, int recordCount, int keyReuse, Function keySize, 39 | Function valueSize) { 40 | int snapshotId = startingSnapshotId; 41 | List tuples = new ArrayList(recordCount); 42 | 43 | for (int i = 0; i < recordCount; i++) { 44 | ByteBuffer key = testDataGenerator.testKey(keySize.apply(), keyReuse); 45 | ByteBuffer value = testDataGenerator.testValue(valueSize.apply()); 46 | tuples.add(new Tuple(new Key(key, snapshotId), new Value(value))); 47 | snapshotId++; 48 | } 49 | 50 | Collections.sort(tuples); 51 | 52 | return tuples; 53 | } 54 | 55 | 56 | public List testRecords(int startingSnapshotId, int recordCount, int keyReuse, final int keySize, 57 | final int valueSize) { 58 | return testRecords(startingSnapshotId, recordCount, keyReuse, new Function() { 59 | @Override 60 | public Integer apply() { 61 | return keySize; 62 | } 63 | }, new Function() { 64 | @Override 65 | public Integer apply() { 66 | return valueSize; 67 | } 68 | } 69 | ); 70 | } 71 | 72 | public List testRecords(int recordCount, int keyReuse, int keySize, int valueSize) { 73 | return testRecords(0, recordCount, keyReuse, keySize, valueSize); 74 | } 75 | 76 | public List testRecords(int recordCount, int keyReuse) { 77 | return testRecords(recordCount, keyReuse, 16, 100); 78 | } 79 | 80 | public static List latest(List tuples, long snapshotId) { 81 | SortedMap latestRecordMap = new TreeMap(); 82 | 83 | for (Tuple tuple : tuples) { 84 | Tuple existing = latestRecordMap.get(tuple.key().data()); 85 | 86 | if (existing == null || tuple.key().snapshotId() > existing.key().snapshotId()) { 87 | latestRecordMap.put(tuple.key().data(), tuple); 88 | } 89 | } 90 | 91 | return new ArrayList(latestRecordMap.values()); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/helper/CompareHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.helper; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Record; 21 | import org.junit.Assert; 22 | 23 | import java.util.Iterator; 24 | 25 | public class CompareHelper { 26 | 27 | public static void compareKeyValue(Iterator tupleIterator, Iterator recordIterator) { 28 | while (tupleIterator.hasNext()) { 29 | compareKeyValue(tupleIterator.next(), recordIterator.next()); 30 | } 31 | 32 | Assert.assertFalse("Record iterator has next", recordIterator.hasNext()); 33 | } 34 | 35 | public static void compareKeyValue(Tuple tuple, Record record) { 36 | Assert.assertNotNull("Record is not null", record); 37 | Assert.assertNotNull("Tuple is not null", record); 38 | Assert.assertEquals("Keys match", tuple.key().data(), record.key()); 39 | Assert.assertEquals("Values match", tuple.value().data(), record.value()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/helper/PerformanceHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.helper; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | 22 | import java.util.concurrent.TimeUnit; 23 | 24 | public class PerformanceHelper { 25 | 26 | public static ConsoleReporter consoleReporter(MetricRegistry metrics) { 27 | return ConsoleReporter.forRegistry(metrics).convertDurationsTo(TimeUnit.MICROSECONDS).convertRatesTo(TimeUnit 28 | .SECONDS).build(); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/helper/StopWatch.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.helper; 18 | 19 | public final class StopWatch { 20 | 21 | private final long startTime; 22 | 23 | public StopWatch() { 24 | startTime = System.nanoTime(); 25 | } 26 | 27 | public long elapsed() { 28 | return System.nanoTime() - startTime; 29 | } 30 | 31 | public double elapsedMicros() { 32 | return (double) elapsed() / 1000; 33 | } 34 | 35 | public double elapsedMillis() { 36 | return (double) elapsed() / 1000000; 37 | } 38 | 39 | public double elapsedSeconds() { 40 | return (double) elapsed() / 1000000000; 41 | } 42 | 43 | public double elapsedMinutes() { 44 | return elapsedSeconds() / 60; 45 | } 46 | 47 | public static StopWatch start() { 48 | return new StopWatch(); 49 | } 50 | } -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/helper/TestFileHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.helper; 18 | 19 | import java.io.IOException; 20 | import java.nio.file.DirectoryStream; 21 | import java.nio.file.Files; 22 | import java.nio.file.Path; 23 | import java.nio.file.Paths; 24 | 25 | public class TestFileHelper { 26 | 27 | public static final Path TEMP_PATH = Paths.get("/tmp/heftytest"); 28 | 29 | public static void cleanUpTestFiles() throws IOException { 30 | DirectoryStream filePaths = Files.newDirectoryStream(TEMP_PATH); 31 | 32 | for (Path f : filePaths) { 33 | if (Files.exists(f)) { 34 | Files.delete(f); 35 | } 36 | } 37 | } 38 | 39 | public static void createTestDirectory() throws IOException { 40 | if (!Files.exists(TEMP_PATH)) { 41 | Files.createDirectory(TEMP_PATH); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/integration/DeleteTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.integration; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.HeftyDB; 22 | import com.jordanwilliams.heftydb.test.base.ParameterizedIntegrationTest; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | import java.io.IOException; 27 | import java.nio.ByteBuffer; 28 | import java.util.List; 29 | import java.util.Random; 30 | 31 | public class DeleteTest extends ParameterizedIntegrationTest { 32 | 33 | public DeleteTest(List tuples, Config config) throws IOException { 34 | super(tuples, config); 35 | } 36 | 37 | @Test 38 | public void deleteTest() throws Exception { 39 | writeRecords(); 40 | Random random = new Random(System.nanoTime()); 41 | 42 | db = HeftyDB.open(config); 43 | 44 | ByteBuffer keyToDelete = tuples.get(random.nextInt(tuples.size())).key().data(); 45 | db.delete(keyToDelete); 46 | Assert.assertNull("Key was deleted", db.get(keyToDelete)); 47 | 48 | db.close(); 49 | } 50 | } -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/integration/IteratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.integration; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.HeftyDB; 22 | import com.jordanwilliams.heftydb.db.Record; 23 | import com.jordanwilliams.heftydb.db.Snapshot; 24 | import com.jordanwilliams.heftydb.test.base.ParameterizedIntegrationTest; 25 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 26 | import com.jordanwilliams.heftydb.test.helper.CompareHelper; 27 | import org.junit.Test; 28 | 29 | import java.io.IOException; 30 | import java.util.Collections; 31 | import java.util.Iterator; 32 | import java.util.List; 33 | 34 | public class IteratorTest extends ParameterizedIntegrationTest { 35 | 36 | public IteratorTest(List tuples, Config config) throws IOException { 37 | super(tuples, config); 38 | writeRecords(); 39 | } 40 | 41 | @Test 42 | public void ascendingIteratorTest() throws Exception { 43 | db = HeftyDB.open(config); 44 | 45 | Iterator dbIterator = db.ascendingIterator(Snapshot.MAX); 46 | Iterator tupleIterator = TupleGenerator.latest(tuples, Long.MAX_VALUE).iterator(); 47 | 48 | CompareHelper.compareKeyValue(tupleIterator, dbIterator); 49 | 50 | db.close(); 51 | } 52 | 53 | @Test 54 | public void descendingIteratorTest() throws Exception { 55 | db = HeftyDB.open(config); 56 | 57 | Iterator dbIterator = db.descendingIterator(Snapshot.MAX); 58 | List latest = TupleGenerator.latest(tuples, Long.MAX_VALUE); 59 | Collections.reverse(latest); 60 | Iterator tupleIterator = latest.iterator(); 61 | 62 | CompareHelper.compareKeyValue(tupleIterator, dbIterator); 63 | 64 | db.close(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/integration/ReadWriteTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.integration; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.HeftyDB; 22 | import com.jordanwilliams.heftydb.db.Record; 23 | import com.jordanwilliams.heftydb.test.base.ParameterizedIntegrationTest; 24 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 25 | import com.jordanwilliams.heftydb.test.helper.CompareHelper; 26 | import org.junit.Test; 27 | 28 | import java.io.IOException; 29 | import java.util.List; 30 | 31 | public class ReadWriteTest extends ParameterizedIntegrationTest { 32 | 33 | public ReadWriteTest(List tuples, Config config) throws IOException { 34 | super(tuples, config); 35 | } 36 | 37 | @Test 38 | public void readWriteTest() throws Exception { 39 | writeRecords(); 40 | 41 | db = HeftyDB.open(config); 42 | 43 | for (Tuple tuple : TupleGenerator.latest(tuples, Long.MAX_VALUE)) { 44 | Record record = db.get(tuple.key().data()); 45 | CompareHelper.compareKeyValue(tuple, record); 46 | } 47 | 48 | db.close(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/db/ReadPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.db; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.ExponentiallyDecayingReservoir; 21 | import com.codahale.metrics.MetricRegistry; 22 | import com.codahale.metrics.Timer; 23 | import com.jordanwilliams.heftydb.compact.CompactionStrategies; 24 | import com.jordanwilliams.heftydb.db.Config; 25 | import com.jordanwilliams.heftydb.db.DB; 26 | import com.jordanwilliams.heftydb.db.HeftyDB; 27 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | import com.jordanwilliams.heftydb.util.ByteBuffers; 30 | 31 | import java.util.Random; 32 | 33 | public class ReadPerformance { 34 | 35 | private static final int RECORD_COUNT = 1 * 1000000; 36 | 37 | public static void main(String[] args) throws Exception { 38 | Random random = new Random(System.nanoTime()); 39 | 40 | Config config = new Config.Builder().directory(TestFileHelper.TEMP_PATH).compactionStrategy 41 | (CompactionStrategies.SIZE_TIERED_COMPACTION_STRATEGY).tableCacheSize(512000000).indexCacheSize 42 | (64000000).maxWriteRate(Integer.MAX_VALUE).build(); 43 | 44 | MetricRegistry metrics = new MetricRegistry(); 45 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 46 | Timer readTimer = metrics.register("reads", new Timer(new ExponentiallyDecayingReservoir())); 47 | 48 | DB db = HeftyDB.open(config); 49 | 50 | db.compact().get(); 51 | 52 | //Read 53 | for (int i = 0; i < RECORD_COUNT * 10; i++) { 54 | String key = random.nextInt(RECORD_COUNT) + ""; 55 | Timer.Context watch = readTimer.time(); 56 | db.get(ByteBuffers.fromString(key)); 57 | watch.stop(); 58 | } 59 | 60 | reporter.report(); 61 | db.logMetrics(); 62 | db.close(); 63 | 64 | System.exit(0); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/db/ReadWritePerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.db; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.ExponentiallyDecayingReservoir; 21 | import com.codahale.metrics.MetricRegistry; 22 | import com.codahale.metrics.Timer; 23 | import com.jordanwilliams.heftydb.compact.CompactionStrategies; 24 | import com.jordanwilliams.heftydb.data.Value; 25 | import com.jordanwilliams.heftydb.db.Config; 26 | import com.jordanwilliams.heftydb.db.DB; 27 | import com.jordanwilliams.heftydb.db.HeftyDB; 28 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 29 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 30 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 31 | import com.jordanwilliams.heftydb.util.ByteBuffers; 32 | 33 | import java.util.Random; 34 | 35 | public class ReadWritePerformance { 36 | 37 | private static final int RECORD_COUNT = 20 * 1000000; 38 | 39 | public static void main(String[] args) throws Exception { 40 | TestFileHelper.createTestDirectory(); 41 | TestFileHelper.cleanUpTestFiles(); 42 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 43 | Value value = new Value(keyValueGenerator.testValue(100)); 44 | Random random = new Random(System.nanoTime()); 45 | 46 | Config config = new Config.Builder().directory(TestFileHelper.TEMP_PATH).compactionStrategy 47 | (CompactionStrategies.FULL_COMPACTION_STRATEGY).memoryTableSize(32768000).tableCacheSize(3276800) 48 | .indexCacheSize(64000000).tableBlockSize(32768).indexBlockSize(65000).maxWriteRate(Integer.MAX_VALUE) 49 | .build(); 50 | 51 | MetricRegistry metrics = new MetricRegistry(); 52 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 53 | 54 | //Write 55 | DB db = HeftyDB.open(config); 56 | 57 | Timer writeTimer = metrics.register("writes", new Timer(new ExponentiallyDecayingReservoir())); 58 | 59 | for (int i = 0; i < RECORD_COUNT; i++) { 60 | value.data().rewind(); 61 | Timer.Context watch = writeTimer.time(); 62 | db.put(ByteBuffers.fromString(i + ""), value.data()); 63 | watch.stop(); 64 | } 65 | 66 | reporter.report(); 67 | 68 | 69 | db.close(); 70 | 71 | metrics = new MetricRegistry(); 72 | reporter = PerformanceHelper.consoleReporter(metrics); 73 | Timer readTimer = metrics.register("reads", new Timer(new ExponentiallyDecayingReservoir())); 74 | 75 | db = HeftyDB.open(config); 76 | 77 | db.compact().get(); 78 | 79 | //Read 80 | for (int i = 0; i < RECORD_COUNT; i++) { 81 | String key = random.nextInt(RECORD_COUNT) + ""; 82 | Timer.Context watch = readTimer.time(); 83 | db.get(ByteBuffers.fromString(key)); 84 | watch.stop(); 85 | } 86 | 87 | reporter.report(); 88 | 89 | db.logMetrics(); 90 | 91 | db.close(); 92 | 93 | TestFileHelper.cleanUpTestFiles(); 94 | 95 | System.exit(0); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/db/ScanPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.db; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Value; 23 | import com.jordanwilliams.heftydb.db.Config; 24 | import com.jordanwilliams.heftydb.db.DB; 25 | import com.jordanwilliams.heftydb.db.HeftyDB; 26 | import com.jordanwilliams.heftydb.db.Record; 27 | import com.jordanwilliams.heftydb.db.Snapshot; 28 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 29 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 30 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 31 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 32 | import com.jordanwilliams.heftydb.util.ByteBuffers; 33 | 34 | import java.util.Iterator; 35 | 36 | public class ScanPerformance { 37 | 38 | private static final int RECORD_COUNT = 5 * 1000000; 39 | 40 | public static void main(String[] args) throws Exception { 41 | MetricRegistry metrics = new MetricRegistry(); 42 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 43 | Timer scanTimer = metrics.timer("scans"); 44 | 45 | TestFileHelper.createTestDirectory(); 46 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 47 | Value value = new Value(keyValueGenerator.testValue(100)); 48 | 49 | Config config = ConfigGenerator.defaultConfig(); 50 | 51 | //Write 52 | final DB db = HeftyDB.open(config); 53 | 54 | for (int i = 0; i < RECORD_COUNT; i++) { 55 | value.data().rewind(); 56 | db.put(ByteBuffers.fromString(i + ""), value.data()); 57 | } 58 | 59 | //Scan 60 | Iterator iterator = db.ascendingIterator(Snapshot.MAX); 61 | 62 | while (iterator.hasNext()) { 63 | Timer.Context watch = scanTimer.time(); 64 | iterator.next(); 65 | watch.stop(); 66 | } 67 | 68 | db.close(); 69 | 70 | reporter.report(); 71 | 72 | TestFileHelper.cleanUpTestFiles(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/db/WritePerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.db; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.compact.CompactionStrategies; 23 | import com.jordanwilliams.heftydb.db.Config; 24 | import com.jordanwilliams.heftydb.db.DB; 25 | import com.jordanwilliams.heftydb.db.HeftyDB; 26 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 27 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | import com.jordanwilliams.heftydb.util.ByteBuffers; 30 | 31 | import java.nio.ByteBuffer; 32 | 33 | public class WritePerformance { 34 | 35 | private static final int RECORD_COUNT = 1 * 1000000; 36 | 37 | public static void main(String[] args) throws Exception { 38 | TestFileHelper.createTestDirectory(); 39 | TestFileHelper.cleanUpTestFiles(); 40 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 41 | ByteBuffer testValueBuffer = keyValueGenerator.testValue(100); 42 | 43 | Config config = new Config.Builder().directory(TestFileHelper.TEMP_PATH).memoryTableSize(16384000) 44 | .tableCacheSize(512000000).indexCacheSize(64000000).tableBlockSize(16384).compactionStrategy 45 | (CompactionStrategies.SIZE_TIERED_COMPACTION_STRATEGY).indexBlockSize(32768).maxWriteRate 46 | (Integer.MAX_VALUE).build(); 47 | 48 | //Write 49 | DB db = HeftyDB.open(config); 50 | 51 | MetricRegistry metrics = new MetricRegistry(); 52 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 53 | Timer writeTimer = metrics.timer("writes"); 54 | 55 | for (int i = 0; i < RECORD_COUNT; i++) { 56 | Timer.Context watch = writeTimer.time(); 57 | db.put(ByteBuffers.fromString(i + ""), testValueBuffer.slice()); 58 | watch.stop(); 59 | } 60 | 61 | reporter.report(); 62 | db.close(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/offheap/BlockCreationPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.offheap; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Key; 23 | import com.jordanwilliams.heftydb.index.IndexBlock; 24 | import com.jordanwilliams.heftydb.index.IndexRecord; 25 | import com.jordanwilliams.heftydb.offheap.SortedByteMap; 26 | import com.jordanwilliams.heftydb.offheap.MemoryPointer; 27 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 28 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 29 | 30 | import java.util.ArrayList; 31 | import java.util.Collections; 32 | import java.util.List; 33 | 34 | public class BlockCreationPerformance { 35 | 36 | public static void main(String[] args) { 37 | MetricRegistry metrics = new MetricRegistry(); 38 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 39 | Timer timer = metrics.timer("blockCreationTime"); 40 | 41 | KeyValueGenerator generator = new KeyValueGenerator(); 42 | List keys = new ArrayList(); 43 | 44 | for (int i = 0; i < 64000; i++) { 45 | keys.add(new Key(generator.testKey(32, 0), i)); 46 | } 47 | 48 | Collections.sort(keys); 49 | 50 | IndexBlock.Builder blockBuilder = new IndexBlock.Builder(); 51 | 52 | for (Key key : keys) { 53 | blockBuilder.addRecord(new IndexRecord(key, 0, 128)); 54 | } 55 | 56 | IndexBlock block = blockBuilder.build(); 57 | MemoryPointer blockPointer = block.memory(); 58 | 59 | int iterations = 10000000; 60 | 61 | for (int i = 0; i < iterations; i++) { 62 | Timer.Context watch = timer.time(); 63 | block = new IndexBlock(new SortedByteMap(blockPointer)); 64 | watch.stop(); 65 | } 66 | 67 | reporter.report(); 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/offheap/MemoryPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.offheap; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.offheap.MemoryAllocator; 23 | import com.jordanwilliams.heftydb.offheap.MemoryPointer; 24 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 25 | 26 | import java.util.Random; 27 | 28 | public class MemoryPerformance { 29 | 30 | public static void main(String[] args) throws Exception { 31 | MetricRegistry metrics = new MetricRegistry(); 32 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 33 | Timer timer = metrics.timer("allocations"); 34 | 35 | Random random = new Random(System.nanoTime()); 36 | int iterations = 1000000; 37 | MemoryPointer[] pointerArray = new MemoryPointer[iterations]; 38 | 39 | for (int i = 0; i < pointerArray.length; i++) { 40 | Timer.Context watch = timer.time(); 41 | pointerArray[i] = MemoryAllocator.allocate(random.nextInt(16384)); 42 | watch.stop(); 43 | } 44 | 45 | reporter.report(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/file/FileTablePerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.file; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Key; 23 | import com.jordanwilliams.heftydb.data.Tuple; 24 | import com.jordanwilliams.heftydb.data.Value; 25 | import com.jordanwilliams.heftydb.db.Config; 26 | import com.jordanwilliams.heftydb.index.IndexBlock; 27 | import com.jordanwilliams.heftydb.metrics.Metrics; 28 | import com.jordanwilliams.heftydb.state.Paths; 29 | import com.jordanwilliams.heftydb.table.file.FileTable; 30 | import com.jordanwilliams.heftydb.table.file.FileTableWriter; 31 | import com.jordanwilliams.heftydb.table.file.TupleBlock; 32 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 33 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 34 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 35 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 36 | import com.jordanwilliams.heftydb.util.ByteBuffers; 37 | 38 | import java.nio.file.Files; 39 | import java.nio.file.StandardCopyOption; 40 | import java.util.Random; 41 | 42 | public class FileTablePerformance { 43 | 44 | private static final int RECORD_COUNT = 5 * 1000000; 45 | 46 | public static void main(String[] args) throws Exception { 47 | MetricRegistry metrics = new MetricRegistry(); 48 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 49 | Timer writeTimer = metrics.timer("writes"); 50 | Timer readTimer = metrics.timer("reads"); 51 | Config config = ConfigGenerator.performanceConfig(); 52 | 53 | TestFileHelper.createTestDirectory(); 54 | TestFileHelper.cleanUpTestFiles(); 55 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 56 | Value value = new Value(keyValueGenerator.testValue(100)); 57 | Paths paths = ConfigGenerator.testPaths(); 58 | 59 | System.out.println("Writing file table"); 60 | 61 | FileTableWriter fileTableWriter = FileTableWriter.open(1, paths, RECORD_COUNT, 32768, 8192, 1); 62 | for (int i = 0; i < RECORD_COUNT; i++) { 63 | value.data().rewind(); 64 | Timer.Context watch = writeTimer.time(); 65 | fileTableWriter.write(new Tuple(new Key(ByteBuffers.fromString(i + ""), i), value)); 66 | watch.stop(); 67 | } 68 | 69 | fileTableWriter.finish(); 70 | 71 | Files.move(paths.tempPath(1), paths.tablePath(1), StandardCopyOption.ATOMIC_MOVE); 72 | 73 | System.out.println("Reading file table"); 74 | 75 | FileTable fileTable = FileTable.open(1, paths, new TupleBlock.Cache(512000000, new Metrics(config)), 76 | new IndexBlock.Cache(16384000, new Metrics(config)), new Metrics(config)); 77 | 78 | Random random = new Random(System.nanoTime()); 79 | int iterations = 2 * 1000000; 80 | 81 | for (int i = 0; i < iterations; i++) { 82 | Timer.Context watch = readTimer.time(); 83 | fileTable.get(new Key(ByteBuffers.fromString(random.nextInt(RECORD_COUNT) + ""), Long.MAX_VALUE)); 84 | watch.stop(); 85 | } 86 | 87 | reporter.report(); 88 | 89 | TestFileHelper.cleanUpTestFiles(); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/file/IndexBlockPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.file; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Key; 23 | import com.jordanwilliams.heftydb.index.IndexBlock; 24 | import com.jordanwilliams.heftydb.index.IndexRecord; 25 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 26 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 27 | 28 | import java.util.ArrayList; 29 | import java.util.Collections; 30 | import java.util.List; 31 | import java.util.Random; 32 | 33 | public class IndexBlockPerformance { 34 | 35 | public static void main(String[] args) { 36 | MetricRegistry metrics = new MetricRegistry(); 37 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 38 | Timer timer = metrics.timer("reads"); 39 | 40 | KeyValueGenerator generator = new KeyValueGenerator(); 41 | List keys = new ArrayList(); 42 | 43 | for (int i = 0; i < 64000; i++) { 44 | keys.add(new Key(generator.testKey(32, 0), i)); 45 | } 46 | 47 | Collections.sort(keys); 48 | 49 | IndexBlock.Builder blockBuilder = new IndexBlock.Builder(); 50 | for (Key key : keys) { 51 | blockBuilder.addRecord(new IndexRecord(key, 0, 128)); 52 | } 53 | 54 | IndexBlock block = blockBuilder.build(); 55 | 56 | Random random = new Random(System.nanoTime()); 57 | int iterations = 10000000; 58 | 59 | for (int i = 0; i < iterations; i++) { 60 | Timer.Context watch = timer.time(); 61 | block.get(keys.get(random.nextInt(keys.size()))); 62 | watch.stop(); 63 | } 64 | 65 | reporter.report(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/file/IndexPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.index.Index; 22 | import com.jordanwilliams.heftydb.index.IndexBlock; 23 | import com.jordanwilliams.heftydb.metrics.Metrics; 24 | import com.jordanwilliams.heftydb.state.Paths; 25 | import com.jordanwilliams.heftydb.table.file.FileTableWriter; 26 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 27 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | 30 | import java.util.List; 31 | import java.util.Random; 32 | 33 | public class IndexPerformance { 34 | 35 | public static void main(String[] args) throws Exception { 36 | TestFileHelper.createTestDirectory(); 37 | TupleGenerator generator = new TupleGenerator(); 38 | List tuples = generator.testRecords(1, 500000, 20, 16, 100); 39 | 40 | Paths paths = ConfigGenerator.testPaths(); 41 | Config config = ConfigGenerator.testConfig(); 42 | FileTableWriter fileTableWriter = FileTableWriter.open(1, paths, 500000, 32000, 32000, 1); 43 | for (Tuple tuple : tuples) { 44 | fileTableWriter.write(tuple); 45 | } 46 | 47 | fileTableWriter.finish(); 48 | 49 | Index index = Index.open(1, paths, new IndexBlock.Cache(4096000, new Metrics(config)), new Metrics(config)); 50 | 51 | Random random = new Random(System.nanoTime()); 52 | int iterations = 1000000; 53 | 54 | for (int i = 0; i < iterations; i++) { 55 | index.get(tuples.get(random.nextInt(tuples.size())).key()); 56 | } 57 | 58 | TestFileHelper.cleanUpTestFiles(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/file/RecordBlockPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.file; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Tuple; 23 | import com.jordanwilliams.heftydb.table.file.TupleBlock; 24 | import com.jordanwilliams.heftydb.test.generator.TupleGenerator; 25 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 26 | 27 | import java.util.List; 28 | import java.util.Random; 29 | 30 | public class RecordBlockPerformance { 31 | 32 | public static void main(String[] args) { 33 | MetricRegistry metrics = new MetricRegistry(); 34 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 35 | Timer timer = metrics.timer("reads"); 36 | TupleGenerator generator = new TupleGenerator(); 37 | List tuples = generator.testRecords(1, 64000, 20, 16, 100); 38 | 39 | TupleBlock.Builder blockBuilder = new TupleBlock.Builder(); 40 | for (Tuple tuple : tuples) { 41 | blockBuilder.addRecord(tuple); 42 | } 43 | 44 | TupleBlock block = blockBuilder.build(); 45 | 46 | Random random = new Random(System.nanoTime()); 47 | int iterations = 10000000; 48 | 49 | for (int i = 0; i < iterations; i++) { 50 | Timer.Context watch = timer.time(); 51 | block.get(tuples.get(random.nextInt(tuples.size())).key()); 52 | watch.stop(); 53 | } 54 | 55 | reporter.report(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/file/TableBloomFilterPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.file; 18 | 19 | import com.google.common.hash.BloomFilter; 20 | import com.google.common.hash.Funnel; 21 | import com.google.common.hash.PrimitiveSink; 22 | import com.jordanwilliams.heftydb.data.Key; 23 | import com.jordanwilliams.heftydb.data.Value; 24 | import com.jordanwilliams.heftydb.state.Paths; 25 | import com.jordanwilliams.heftydb.table.file.TableBloomFilter; 26 | import com.jordanwilliams.heftydb.table.file.TableBloomFilterWriter; 27 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 28 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 29 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 30 | import com.jordanwilliams.heftydb.util.ByteBuffers; 31 | 32 | public class TableBloomFilterPerformance { 33 | 34 | private static final int RECORD_COUNT = 5 * 1000000; 35 | 36 | public static void main(String[] args) throws Exception { 37 | TestFileHelper.createTestDirectory(); 38 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 39 | Value value = new Value(keyValueGenerator.testValue(100)); 40 | 41 | System.out.println("Writing bloom filter"); 42 | 43 | Paths paths = ConfigGenerator.testPaths(); 44 | TableBloomFilterWriter filterWriter = TableBloomFilterWriter.open(1, paths, RECORD_COUNT); 45 | BloomFilter guavaFilter = BloomFilter.create(new Funnel() { 46 | @Override 47 | public void funnel(Key key, PrimitiveSink primitiveSink) { 48 | primitiveSink.putBytes(key.data().array()); 49 | } 50 | }, RECORD_COUNT, 0.01); 51 | 52 | for (int i = 0; i < RECORD_COUNT; i++) { 53 | value.data().rewind(); 54 | filterWriter.write(new Key(ByteBuffers.fromString(i + ""), i)); 55 | guavaFilter.put(new Key(ByteBuffers.fromString(i + ""), i)); 56 | } 57 | 58 | filterWriter.finish(); 59 | 60 | System.out.println("Reading bloom filter"); 61 | 62 | TableBloomFilter tableBloomFilter = TableBloomFilter.read(1, paths); 63 | 64 | double hits = 0; 65 | double misses = 0; 66 | 67 | double ghits = 0; 68 | double gmisses = 0; 69 | 70 | for (int i = RECORD_COUNT * 2; i > RECORD_COUNT; i--) { 71 | if (tableBloomFilter.mightContain(new Key(ByteBuffers.fromString(i + ""), i))) { 72 | hits++; 73 | } else { 74 | misses++; 75 | } 76 | 77 | if (guavaFilter.mightContain(new Key(ByteBuffers.fromString(i + ""), i))) { 78 | ghits++; 79 | } else { 80 | gmisses++; 81 | } 82 | } 83 | 84 | System.out.println("False positive rate: " + hits / (hits + misses)); 85 | System.out.println("Guava positive rate: " + ghits / (ghits + gmisses)); 86 | 87 | TestFileHelper.cleanUpTestFiles(); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/table/memory/MemoryTablePerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.table.memory; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Key; 23 | import com.jordanwilliams.heftydb.data.Tuple; 24 | import com.jordanwilliams.heftydb.data.Value; 25 | import com.jordanwilliams.heftydb.table.memory.MemoryTable; 26 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 27 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | import com.jordanwilliams.heftydb.util.ByteBuffers; 30 | 31 | import java.util.Random; 32 | 33 | public class MemoryTablePerformance { 34 | 35 | private static final int RECORD_COUNT = 1000000; 36 | 37 | public static void main(String[] args) throws Exception { 38 | MetricRegistry metrics = new MetricRegistry(); 39 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 40 | Timer writeTimer = metrics.timer("writes"); 41 | Timer readTimer = metrics.timer("reads"); 42 | 43 | TestFileHelper.createTestDirectory(); 44 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 45 | Value value = new Value(keyValueGenerator.testValue(100)); 46 | 47 | MemoryTable memTable = new MemoryTable(1); 48 | 49 | for (int i = 0; i < RECORD_COUNT; i++) { 50 | Timer.Context watch = writeTimer.time(); 51 | memTable.put(new Tuple(new Key(ByteBuffers.fromString(i + ""), i), value)); 52 | watch.stop(); 53 | } 54 | 55 | TestFileHelper.cleanUpTestFiles(); 56 | 57 | Random random = new Random(System.nanoTime()); 58 | int iterations = 10000000; 59 | 60 | for (int i = 0; i < iterations; i++) { 61 | Timer.Context watch = readTimer.time(); 62 | memTable.get(new Key(ByteBuffers.fromString(random.nextInt(RECORD_COUNT) + ""), Long.MAX_VALUE)); 63 | watch.stop(); 64 | } 65 | 66 | reporter.report(); 67 | 68 | TestFileHelper.cleanUpTestFiles(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/performance/write/RecordWriterPerformance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.performance.write; 18 | 19 | import com.codahale.metrics.ConsoleReporter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.codahale.metrics.Timer; 22 | import com.jordanwilliams.heftydb.data.Value; 23 | import com.jordanwilliams.heftydb.db.DBState; 24 | import com.jordanwilliams.heftydb.metrics.Metrics; 25 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 26 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 27 | import com.jordanwilliams.heftydb.test.helper.PerformanceHelper; 28 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 29 | import com.jordanwilliams.heftydb.util.ByteBuffers; 30 | import com.jordanwilliams.heftydb.write.TableWriter; 31 | 32 | public class RecordWriterPerformance { 33 | 34 | private static final int RECORD_COUNT = 5 * 1000000; 35 | 36 | public static void main(String[] args) throws Exception { 37 | MetricRegistry metrics = new MetricRegistry(); 38 | ConsoleReporter reporter = PerformanceHelper.consoleReporter(metrics); 39 | Timer timer = metrics.timer("writes"); 40 | 41 | TestFileHelper.createTestDirectory(); 42 | KeyValueGenerator keyValueGenerator = new KeyValueGenerator(); 43 | Value value = new Value(keyValueGenerator.testValue(100)); 44 | 45 | DBState state = ConfigGenerator.perfState(); 46 | TableWriter tableWriter = new TableWriter(state.config(), state.paths(), state.tables(), state.snapshots(), 47 | state.caches(), new Metrics(state.config())); 48 | 49 | for (int i = 0; i < RECORD_COUNT; i++) { 50 | value.data().rewind(); 51 | Timer.Context watch = timer.time(); 52 | tableWriter.write(ByteBuffers.fromString(i + ""), value.data(), false); 53 | watch.stop(); 54 | } 55 | 56 | reporter.report(); 57 | tableWriter.close(); 58 | 59 | TestFileHelper.cleanUpTestFiles(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/data/TupleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.data; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.nio.ByteBuffer; 26 | 27 | public class TupleTest { 28 | 29 | private static final ByteBuffer TEST_KEY = ByteBuffer.wrap("I am a test key".getBytes()); 30 | private static final ByteBuffer TEST_VALUE = ByteBuffer.wrap("I am a test value".getBytes()); 31 | 32 | @Test 33 | public void testSerialization() { 34 | Tuple tuple = new Tuple(new Key(TEST_KEY, 1), new Value(TEST_VALUE)); 35 | ByteBuffer serialized = ByteBuffer.allocate(Tuple.SERIALIZER.size(tuple)); 36 | Tuple.SERIALIZER.serialize(tuple, serialized); 37 | Tuple deserializedTuple = Tuple.SERIALIZER.deserialize(serialized); 38 | 39 | Assert.assertEquals("Deserialized tuples match", tuple, deserializedTuple); 40 | } 41 | 42 | @Test 43 | public void testTombstoneSerialization() { 44 | Tuple tuple = new Tuple(new Key(TEST_KEY, 1), Value.TOMBSTONE_VALUE); 45 | ByteBuffer serialized = ByteBuffer.allocate(Tuple.SERIALIZER.size(tuple)); 46 | Tuple.SERIALIZER.serialize(tuple, serialized); 47 | Tuple deserializedTuple = Tuple.SERIALIZER.deserialize(serialized); 48 | 49 | Assert.assertEquals("Deserialized tombstone tuples match", tuple, deserializedTuple); 50 | Assert.assertTrue("Tuple is a tombstone tuple", deserializedTuple.value().isEmpty()); 51 | } 52 | } -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/db/DBInitializerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.db; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.db.DBInitializer; 22 | import com.jordanwilliams.heftydb.db.DBState; 23 | import com.jordanwilliams.heftydb.io.Throttle; 24 | import com.jordanwilliams.heftydb.metrics.Metrics; 25 | import com.jordanwilliams.heftydb.state.Paths; 26 | import com.jordanwilliams.heftydb.table.file.FileTableWriter; 27 | import com.jordanwilliams.heftydb.test.base.TupleTest; 28 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 29 | import com.jordanwilliams.heftydb.write.CommitLogWriter; 30 | import org.junit.Assert; 31 | import org.junit.Test; 32 | 33 | import java.util.List; 34 | 35 | public class DBInitializerTest extends TupleTest { 36 | 37 | @Test 38 | public void defaultStateTest() throws Exception { 39 | Config config = ConfigGenerator.testConfig(); 40 | DBState state = new DBInitializer(config, new Metrics(config)).initialize(); 41 | Assert.assertTrue("No tables", state.tables().count() == 0); 42 | Assert.assertEquals("No Max Snapshot Id", 0, state.snapshots().currentId()); 43 | } 44 | 45 | @Test 46 | public void existingStateTest() throws Exception { 47 | Paths paths = ConfigGenerator.testPaths(); 48 | Config config = ConfigGenerator.testConfig(); 49 | FileTableWriter.Task writerTask = new FileTableWriter.Task(1, 1, paths, config, tuples.iterator(), 50 | tuples.size(), null, Throttle.MAX); 51 | writerTask.run(); 52 | 53 | DBState state = new DBInitializer(config, new Metrics(config)).initialize(); 54 | Assert.assertEquals("Should be 1 table", 1, state.tables().count()); 55 | Assert.assertEquals("Should be 100 as the max snapshot id", 100, state.snapshots().currentId()); 56 | } 57 | 58 | @Test 59 | public void existingStateLogTest() throws Exception { 60 | Paths paths = ConfigGenerator.testPaths(); 61 | Config config = ConfigGenerator.testConfig(); 62 | FileTableWriter.Task writerTask = new FileTableWriter.Task(1, 1, paths, config, tuples.iterator(), 63 | tuples.size(), null, Throttle.MAX); 64 | writerTask.run(); 65 | 66 | CommitLogWriter log = CommitLogWriter.open(2, paths); 67 | List moreTestTuples = generateMoreTestRecords(101); 68 | for (Tuple tuple : moreTestTuples) { 69 | log.append(tuple, false); 70 | } 71 | 72 | log.close(); 73 | 74 | DBState state = new DBInitializer(config, new Metrics(config)).initialize(); 75 | Assert.assertEquals("Should be 2 tables", 2, state.tables().count()); 76 | Assert.assertEquals("Should be 200 as the max snapshot id", 200, state.snapshots().currentId()); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/index/IndexBlockRandomTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.index; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.index.IndexBlock; 21 | import com.jordanwilliams.heftydb.index.IndexRecord; 22 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | import java.util.ArrayList; 27 | import java.util.Iterator; 28 | import java.util.List; 29 | 30 | public class IndexBlockRandomTest extends ParameterizedTupleTest { 31 | 32 | private final IndexBlock indexBlock; 33 | private final List indexRecords = new ArrayList(); 34 | 35 | public IndexBlockRandomTest(List testTuples) { 36 | super(testTuples); 37 | 38 | int count = 0; 39 | 40 | IndexBlock.Builder indexBlockBuilder = new IndexBlock.Builder(); 41 | 42 | for (Tuple tuple : tuples) { 43 | IndexRecord indexRecord = new IndexRecord(tuple.key(), count, 128); 44 | indexRecords.add(indexRecord); 45 | indexBlockBuilder.addRecord(indexRecord); 46 | count++; 47 | } 48 | 49 | this.indexBlock = indexBlockBuilder.build(); 50 | } 51 | 52 | @Test 53 | public void iteratorTest() { 54 | Iterator indexRecordIterator = indexRecords.iterator(); 55 | Iterator indexBlockIterator = indexBlock.ascendingIterator(); 56 | 57 | while (indexRecordIterator.hasNext()) { 58 | IndexRecord recordNext = indexRecordIterator.next(); 59 | IndexRecord blockNext = indexBlockIterator.next(); 60 | Assert.assertEquals("Records match", recordNext, blockNext); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/index/IndexTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.index; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.db.Config; 21 | import com.jordanwilliams.heftydb.index.Index; 22 | import com.jordanwilliams.heftydb.index.IndexBlock; 23 | import com.jordanwilliams.heftydb.index.IndexRecord; 24 | import com.jordanwilliams.heftydb.index.IndexWriter; 25 | import com.jordanwilliams.heftydb.metrics.Metrics; 26 | import com.jordanwilliams.heftydb.state.Paths; 27 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 28 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 29 | import org.junit.Assert; 30 | import org.junit.Test; 31 | 32 | import java.io.IOException; 33 | import java.util.ArrayList; 34 | import java.util.List; 35 | 36 | public class IndexTest extends ParameterizedTupleTest { 37 | 38 | private final Index index; 39 | private final List indexRecords = new ArrayList(); 40 | 41 | public IndexTest(List testTuples) throws Exception { 42 | super(testTuples); 43 | this.index = createIndex(); 44 | } 45 | 46 | @Test 47 | public void getTest() throws IOException { 48 | int count = 0; 49 | 50 | for (Tuple tuple : tuples) { 51 | IndexRecord indexRecord = index.get(tuple.key()); 52 | Assert.assertEquals("Index blocks are found", count, indexRecord.blockOffset()); 53 | count++; 54 | } 55 | 56 | index.close(); 57 | } 58 | 59 | private Index createIndex() throws IOException { 60 | Paths paths = ConfigGenerator.testPaths(); 61 | Config config = ConfigGenerator.testConfig(); 62 | IndexWriter indexWriter = IndexWriter.open(1, paths, 512); 63 | 64 | int count = 0; 65 | 66 | for (Tuple tuple : tuples) { 67 | IndexRecord indexRecord = new IndexRecord(tuple.key(), count, 128); 68 | indexRecords.add(indexRecord); 69 | indexWriter.write(indexRecord); 70 | count++; 71 | } 72 | 73 | indexWriter.finish(); 74 | 75 | return Index.open(1, paths, new IndexBlock.Cache(1024000, new Metrics(config)), new Metrics(config)); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/io/FileIOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.io; 18 | 19 | 20 | import com.jordanwilliams.heftydb.io.AppendChannelFile; 21 | import com.jordanwilliams.heftydb.io.AppendFile; 22 | import com.jordanwilliams.heftydb.io.ImmutableChannelFile; 23 | import com.jordanwilliams.heftydb.io.ImmutableFile; 24 | import com.jordanwilliams.heftydb.test.base.FileTest; 25 | import com.jordanwilliams.heftydb.test.helper.TestFileHelper; 26 | import org.junit.Assert; 27 | import org.junit.Test; 28 | 29 | import java.io.IOException; 30 | import java.nio.ByteBuffer; 31 | import java.nio.file.Path; 32 | 33 | public class FileIOTest extends FileTest { 34 | 35 | private static final ByteBuffer TEST_BYTES = ByteBuffer.wrap("I am some very impressive test data".getBytes()); 36 | private static final ByteBuffer MORE_TEST_BYTES = ByteBuffer.wrap("Test data is very interesting".getBytes()); 37 | 38 | private final Path testFile = TestFileHelper.TEMP_PATH.resolve("testfile"); 39 | 40 | @Test 41 | public void bufferedAppendOffsetTest() throws IOException { 42 | AppendFile file = AppendChannelFile.open(testFile); 43 | 44 | long[] offsets = new long[100]; 45 | 46 | for (int i = 0; i < 100; i++) { 47 | TEST_BYTES.rewind(); 48 | offsets[i] = file.appendInt(TEST_BYTES.capacity()); 49 | file.append(TEST_BYTES); 50 | } 51 | 52 | for (int i = 0; i < 100; i++) { 53 | long offset = offsets[i]; 54 | Assert.assertEquals("Offset is correct", 0, offset % 39); 55 | } 56 | } 57 | 58 | @Test 59 | public void mutableDataFileTest() throws IOException { 60 | TEST_BYTES.rewind(); 61 | MORE_TEST_BYTES.rewind(); 62 | 63 | AppendFile file = AppendChannelFile.open(testFile); 64 | 65 | file.append(TEST_BYTES); 66 | file.append(MORE_TEST_BYTES); 67 | 68 | TEST_BYTES.rewind(); 69 | file.append(TEST_BYTES); 70 | file.close(); 71 | 72 | ByteBuffer readBuffer = ByteBuffer.allocate(TEST_BYTES.capacity()); 73 | 74 | ImmutableFile readFile = ImmutableChannelFile.open(testFile); 75 | readFile.read(readBuffer, 0); 76 | 77 | TEST_BYTES.rewind(); 78 | readBuffer.rewind(); 79 | 80 | Assert.assertEquals("Read bytes", TEST_BYTES, readBuffer); 81 | Assert.assertEquals("File size", (TEST_BYTES.capacity() * 2) + MORE_TEST_BYTES.capacity(), readFile.size()); 82 | } 83 | 84 | @Test 85 | public void mutableDataFilePrimitiveTest() throws IOException { 86 | AppendFile file = AppendChannelFile.open(testFile); 87 | file.appendInt(4); 88 | file.appendLong(8); 89 | file.close(); 90 | 91 | ImmutableFile readFile = ImmutableChannelFile.open(testFile); 92 | Assert.assertEquals("Values match", 4, readFile.readInt(0)); 93 | Assert.assertEquals("Values match", 8, readFile.readLong(4)); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/offheap/BitSetTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.offheap; 18 | 19 | import com.jordanwilliams.heftydb.offheap.BitSet; 20 | import com.jordanwilliams.heftydb.util.Sizes; 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | import java.util.Random; 25 | 26 | public class BitSetTest { 27 | 28 | @Test 29 | public void getSetTest() { 30 | BitSet.Builder testSetBuilder = new BitSet.Builder(256, Sizes.INT_SIZE); 31 | boolean[] values = new boolean[256]; 32 | 33 | Random random = new Random(System.nanoTime()); 34 | 35 | for (int i = 0; i < 256; i++) { 36 | boolean nextValue = random.nextBoolean(); 37 | testSetBuilder.set(i, nextValue); 38 | values[i] = nextValue; 39 | } 40 | 41 | BitSet testSet = testSetBuilder.build(); 42 | 43 | for (int i = 0; i < 256; i++) { 44 | Assert.assertEquals("Values match", values[i], testSet.get(i)); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/offheap/BloomFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.offheap; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.offheap.BloomFilter; 21 | import com.jordanwilliams.heftydb.test.generator.KeyValueGenerator; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | public class BloomFilterTest { 29 | 30 | @Test 31 | public void readWriteTest() { 32 | Set testKeys = testKeys(); 33 | BloomFilter.Builder testFilterBuilder = new BloomFilter.Builder(1000, 0.01); 34 | 35 | for (Key key : testKeys) { 36 | testFilterBuilder.put(key); 37 | } 38 | 39 | BloomFilter testFilter = testFilterBuilder.build(); 40 | 41 | for (Key key : testKeys) { 42 | Assert.assertTrue("Key is in filter", testFilter.mightContain(key)); 43 | } 44 | } 45 | 46 | private static Set testKeys() { 47 | KeyValueGenerator generator = new KeyValueGenerator(); 48 | Set testDataSet = new HashSet(); 49 | 50 | for (int i = 0; i < 1000; i++) { 51 | testDataSet.add(new Key(generator.testKey(255, 0), i)); 52 | } 53 | 54 | return testDataSet; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/offheap/MemoryPointerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.offheap; 18 | 19 | import com.jordanwilliams.heftydb.offheap.MemoryAllocator; 20 | import com.jordanwilliams.heftydb.offheap.MemoryPointer; 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class MemoryPointerTest { 25 | 26 | @Test 27 | public void retainReleaseTest() { 28 | MemoryPointer pointer = MemoryAllocator.allocate(256); 29 | 30 | pointer.retain(); 31 | pointer.retain(); 32 | 33 | pointer.release(); 34 | pointer.release(); 35 | pointer.release(); 36 | 37 | Assert.assertTrue("MemoryPointer has been freed", pointer.isFree()); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/offheap/SortedByteMapTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.offheap; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.offheap.SortedByteMap; 21 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.util.Iterator; 26 | import java.util.List; 27 | 28 | public class SortedByteMapTest extends ParameterizedTupleTest { 29 | 30 | private final SortedByteMap sortedByteMap; 31 | 32 | public SortedByteMapTest(List testTuples) { 33 | super(testTuples); 34 | 35 | SortedByteMap.Builder byteMapBuilder = new SortedByteMap.Builder(); 36 | 37 | for (Tuple tuple : tuples) { 38 | byteMapBuilder.add(tuple.key(), tuple.value()); 39 | } 40 | 41 | this.sortedByteMap = byteMapBuilder.build(); 42 | } 43 | 44 | @Test 45 | public void getTest() { 46 | for (int i = 0; i < tuples.size(); i++) { 47 | SortedByteMap.Entry entry = sortedByteMap.get(i); 48 | Tuple tuple = tuples.get(i); 49 | Assert.assertEquals("Keys match", tuple.key(), entry.key()); 50 | Assert.assertEquals("Values match", tuple.value(), entry.value()); 51 | } 52 | } 53 | 54 | @Test 55 | public void floorIndexMatchTest() { 56 | for (int i = 0; i < tuples.size(); i++) { 57 | Tuple tuple = tuples.get(i); 58 | int floorIndex = sortedByteMap.floorIndex(tuple.key()); 59 | SortedByteMap.Entry entry = sortedByteMap.get(floorIndex); 60 | Assert.assertEquals("Keys match", tuple.key(), entry.key()); 61 | Assert.assertEquals("Values match", tuple.value(), entry.value()); 62 | } 63 | } 64 | 65 | 66 | @Test 67 | public void iteratorTest() { 68 | Iterator recordIterator = tuples.iterator(); 69 | Iterator byteMapIterator = sortedByteMap.iterator(); 70 | 71 | while (recordIterator.hasNext()) { 72 | Tuple tupleNext = recordIterator.next(); 73 | SortedByteMap.Entry byteMapNext = byteMapIterator.next(); 74 | 75 | Assert.assertEquals("Records match", tupleNext, new Tuple(byteMapNext.key(), byteMapNext.value())); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/offheap/UnsafeAllocatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.offheap; 18 | 19 | 20 | import com.jordanwilliams.heftydb.offheap.allocator.UnsafeAllocator; 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class UnsafeAllocatorTest { 25 | 26 | @Test 27 | public void allocateFreeTest() { 28 | UnsafeAllocator unsafeAllocator = new UnsafeAllocator(); 29 | long address = unsafeAllocator.allocate(128); 30 | Assert.assertTrue("Address is non-zero", address != 0); 31 | unsafeAllocator.deallocate(address); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/read/CompactionTupleIteratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.read; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | import com.jordanwilliams.heftydb.read.CompactionTupleIterator; 23 | import com.jordanwilliams.heftydb.util.ByteBuffers; 24 | import com.jordanwilliams.heftydb.util.CloseableIterator; 25 | import org.junit.Assert; 26 | import org.junit.Test; 27 | 28 | import java.nio.ByteBuffer; 29 | import java.util.ArrayList; 30 | import java.util.Iterator; 31 | import java.util.List; 32 | 33 | public class CompactionTupleIteratorTest { 34 | 35 | private static final ByteBuffer KEY_1 = ByteBuffers.fromString("test key 1"); 36 | private static final ByteBuffer KEY_2 = ByteBuffers.fromString("test key 2"); 37 | private static final ByteBuffer KEY_3 = ByteBuffers.fromString("test key 3"); 38 | 39 | private final List sourceTuples = new ArrayList(); 40 | private final List filteredTuples = new ArrayList(); 41 | 42 | public CompactionTupleIteratorTest() { 43 | sourceTuples.add(new Tuple(new Key(KEY_3, 0), Value.TOMBSTONE_VALUE)); 44 | sourceTuples.add(new Tuple(new Key(KEY_1, 1), Value.TOMBSTONE_VALUE)); 45 | sourceTuples.add(new Tuple(new Key(KEY_1, 2), Value.TOMBSTONE_VALUE)); 46 | sourceTuples.add(new Tuple(new Key(KEY_1, 3), Value.TOMBSTONE_VALUE)); 47 | sourceTuples.add(new Tuple(new Key(KEY_2, 4), Value.TOMBSTONE_VALUE)); 48 | sourceTuples.add(new Tuple(new Key(KEY_2, 5), Value.TOMBSTONE_VALUE)); 49 | 50 | filteredTuples.add(new Tuple(new Key(KEY_3, 0), Value.TOMBSTONE_VALUE)); 51 | filteredTuples.add(new Tuple(new Key(KEY_1, 2), Value.TOMBSTONE_VALUE)); 52 | filteredTuples.add(new Tuple(new Key(KEY_1, 3), Value.TOMBSTONE_VALUE)); 53 | filteredTuples.add(new Tuple(new Key(KEY_2, 4), Value.TOMBSTONE_VALUE)); 54 | filteredTuples.add(new Tuple(new Key(KEY_2, 5), Value.TOMBSTONE_VALUE)); 55 | } 56 | 57 | @Test 58 | public void basicFilterTest() { 59 | Iterator filteredIterator = filteredTuples.iterator(); 60 | Iterator versionedIterator = new CompactionTupleIterator(2, new CloseableIterator.Wrapper 61 | (sourceTuples.iterator())); 62 | 63 | while (versionedIterator.hasNext()) { 64 | Assert.assertEquals("Records match", filteredIterator.next(), versionedIterator.next()); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/read/LatestTupleIteratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.read; 18 | 19 | import com.jordanwilliams.heftydb.data.Key; 20 | import com.jordanwilliams.heftydb.data.Tuple; 21 | import com.jordanwilliams.heftydb.data.Value; 22 | import com.jordanwilliams.heftydb.read.LatestTupleIterator; 23 | import com.jordanwilliams.heftydb.util.ByteBuffers; 24 | import com.jordanwilliams.heftydb.util.CloseableIterator; 25 | import org.junit.Assert; 26 | import org.junit.Test; 27 | 28 | import java.nio.ByteBuffer; 29 | import java.util.ArrayList; 30 | import java.util.Iterator; 31 | import java.util.List; 32 | 33 | public class LatestTupleIteratorTest { 34 | 35 | private static final ByteBuffer KEY_1 = ByteBuffers.fromString("test key 1"); 36 | private static final ByteBuffer KEY_2 = ByteBuffers.fromString("test key 2"); 37 | 38 | private final List sourceTuples = new ArrayList(); 39 | private final List filteredTuples = new ArrayList(); 40 | private final List snapshotTuples = new ArrayList(); 41 | 42 | public LatestTupleIteratorTest() { 43 | sourceTuples.add(new Tuple(new Key(KEY_1, 1), Value.TOMBSTONE_VALUE)); 44 | sourceTuples.add(new Tuple(new Key(KEY_1, 2), Value.TOMBSTONE_VALUE)); 45 | sourceTuples.add(new Tuple(new Key(KEY_1, 3), Value.TOMBSTONE_VALUE)); 46 | sourceTuples.add(new Tuple(new Key(KEY_2, 4), Value.TOMBSTONE_VALUE)); 47 | sourceTuples.add(new Tuple(new Key(KEY_2, 5), Value.TOMBSTONE_VALUE)); 48 | 49 | filteredTuples.add(new Tuple(new Key(KEY_1, 3), Value.TOMBSTONE_VALUE)); 50 | filteredTuples.add(new Tuple(new Key(KEY_2, 5), Value.TOMBSTONE_VALUE)); 51 | 52 | snapshotTuples.add(new Tuple(new Key(KEY_1, 3), Value.TOMBSTONE_VALUE)); 53 | snapshotTuples.add(new Tuple(new Key(KEY_2, 4), Value.TOMBSTONE_VALUE)); 54 | } 55 | 56 | @Test 57 | public void filterRecordTest() { 58 | Iterator filteredIterator = filteredTuples.iterator(); 59 | Iterator versionedIterator = new LatestTupleIterator(6, new CloseableIterator.Wrapper 60 | (sourceTuples.iterator())); 61 | 62 | while (versionedIterator.hasNext()) { 63 | Assert.assertEquals("Records match", versionedIterator.next(), filteredIterator.next()); 64 | } 65 | } 66 | 67 | @Test 68 | public void respectSnapshotTest() { 69 | Iterator snapshotIterator = snapshotTuples.iterator(); 70 | Iterator versionedIterator = new LatestTupleIterator(4, new CloseableIterator.Wrapper 71 | (sourceTuples.iterator())); 72 | 73 | while (versionedIterator.hasNext()) { 74 | Assert.assertEquals("Records match", versionedIterator.next(), snapshotIterator.next()); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/read/MergingIteratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.read; 18 | 19 | import com.google.common.primitives.Ints; 20 | import com.jordanwilliams.heftydb.read.MergingIterator; 21 | import com.jordanwilliams.heftydb.util.CloseableIterator; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.util.Iterator; 26 | 27 | public class MergingIteratorTest { 28 | 29 | private static final int[] ARRAY1 = {1, 2, 3, 4, 5, 6}; 30 | private static final int[] ARRAY2 = {1, 4, 5, 7, 9, 10}; 31 | private static final int[] MERGED_ARRAY = {1, 1, 2, 3, 4, 4, 5, 5, 6, 7, 9, 10}; 32 | 33 | @Test 34 | public void mergeTest() { 35 | MergingIterator mergingIterator = new MergingIterator(new CloseableIterator 36 | .Wrapper(Ints.asList(ARRAY1).iterator()), new CloseableIterator.Wrapper(Ints.asList 37 | (ARRAY2).iterator())); 38 | 39 | Iterator mergedIterator = Ints.asList(MERGED_ARRAY).iterator(); 40 | 41 | while (mergedIterator.hasNext()) { 42 | Assert.assertEquals("Merged values match", mergedIterator.next(), mergingIterator.next()); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/table/file/TableBloomFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.state.Paths; 21 | import com.jordanwilliams.heftydb.table.file.TableBloomFilter; 22 | import com.jordanwilliams.heftydb.table.file.TableBloomFilterWriter; 23 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 24 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 25 | import org.junit.Assert; 26 | import org.junit.Test; 27 | 28 | import java.util.List; 29 | 30 | public class TableBloomFilterTest extends ParameterizedTupleTest { 31 | 32 | private final TableBloomFilter bloomFilter; 33 | 34 | public TableBloomFilterTest(List testTuples) throws Exception { 35 | super(testTuples); 36 | 37 | Paths paths = ConfigGenerator.testPaths(); 38 | TableBloomFilterWriter filterWriter = TableBloomFilterWriter.open(1, paths, tuples.size()); 39 | 40 | for (Tuple tuple : tuples) { 41 | filterWriter.write(tuple.key()); 42 | } 43 | 44 | filterWriter.finish(); 45 | 46 | this.bloomFilter = TableBloomFilter.read(1, paths); 47 | } 48 | 49 | @Test 50 | public void mightContainTest() { 51 | for (Tuple tuple : tuples) { 52 | Assert.assertTrue("Filter contains the key", bloomFilter.mightContain(tuple.key())); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/table/file/TupleBlockRandomTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.table.file; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.table.file.TupleBlock; 21 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.util.Iterator; 26 | import java.util.List; 27 | 28 | public class TupleBlockRandomTest extends ParameterizedTupleTest { 29 | 30 | private final TupleBlock tupleBlock; 31 | 32 | public TupleBlockRandomTest(List testTuples) { 33 | super(testTuples); 34 | 35 | TupleBlock.Builder byteMapBuilder = new TupleBlock.Builder(); 36 | 37 | for (Tuple tuple : tuples) { 38 | byteMapBuilder.addRecord(tuple); 39 | } 40 | 41 | this.tupleBlock = byteMapBuilder.build(); 42 | } 43 | 44 | @Test 45 | public void iteratorTest() { 46 | Iterator recordIterator = tuples.iterator(); 47 | Iterator recordBlockIterator = tupleBlock.ascendingIterator(); 48 | 49 | while (recordIterator.hasNext()) { 50 | Tuple tupleNext = recordIterator.next(); 51 | Tuple blockNext = recordBlockIterator.next(); 52 | Assert.assertEquals("Records match", tupleNext, blockNext); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/util/XORShiftRandomTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.util; 18 | 19 | import com.jordanwilliams.heftydb.util.XORShiftRandom; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | public class XORShiftRandomTest { 27 | 28 | 29 | @Test 30 | public void randomTest() { 31 | XORShiftRandom random = new XORShiftRandom(32); 32 | List generatedNumbers = new ArrayList(); 33 | 34 | for (int i = 0; i < 128; i++) { 35 | generatedNumbers.add(random.nextInt(1024)); 36 | } 37 | 38 | random = new XORShiftRandom(32); 39 | 40 | for (int number : generatedNumbers) { 41 | Assert.assertEquals("Generated numbers are equal", number, random.nextInt(1024)); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/jordanwilliams/heftydb/test/unit/write/CommitLogTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Jordan Williams 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.jordanwilliams.heftydb.test.unit.write; 18 | 19 | import com.jordanwilliams.heftydb.data.Tuple; 20 | import com.jordanwilliams.heftydb.state.Paths; 21 | import com.jordanwilliams.heftydb.test.base.ParameterizedTupleTest; 22 | import com.jordanwilliams.heftydb.test.generator.ConfigGenerator; 23 | import com.jordanwilliams.heftydb.write.CommitLog; 24 | import com.jordanwilliams.heftydb.write.CommitLogWriter; 25 | import org.junit.Test; 26 | 27 | import java.io.IOException; 28 | import java.util.Iterator; 29 | import java.util.List; 30 | 31 | public class CommitLogTest extends ParameterizedTupleTest { 32 | 33 | public CommitLogTest(List testTuples) throws Exception { 34 | super(testTuples); 35 | } 36 | 37 | @Test 38 | public void readWriteTest() throws IOException { 39 | Paths paths = ConfigGenerator.testPaths(); 40 | CommitLogWriter log = CommitLogWriter.open(1, paths); 41 | 42 | for (Tuple tuple : tuples) { 43 | log.append(tuple, false); 44 | } 45 | 46 | log.close(); 47 | 48 | CommitLog commitLog = CommitLog.open(1, paths); 49 | Iterator logIterator = commitLog.iterator(); 50 | Iterator recordIterator = tuples.iterator(); 51 | 52 | while (logIterator.hasNext()) { 53 | Tuple next = recordIterator.next(); 54 | Tuple logNext = logIterator.next(); 55 | //Assert.assertEquals("Records match", recordIterator.next(), logIterator.next()); 56 | int x = 1; 57 | } 58 | 59 | commitLog.close(); 60 | } 61 | 62 | } 63 | --------------------------------------------------------------------------------