├── jmh ├── jmh.sh ├── src │ └── main │ │ ├── java │ │ └── org │ │ │ └── deephacks │ │ │ └── rxlmdb │ │ │ ├── Country.java │ │ │ ├── AddressVal.java │ │ │ ├── UserVal.java │ │ │ ├── BigZeroCopyForwardRangeScan.java │ │ │ ├── KeyValueForwardRangeScan.java │ │ │ ├── BigKeyValueForwardRangeScan.java │ │ │ ├── AbstractRxThread.java │ │ │ ├── AbstractPlainThread.java │ │ │ ├── ValsForwardRangeScan.java │ │ │ ├── KeyValueForwardSkipRangeScan.java │ │ │ ├── SbeForwardRangeScan.java │ │ │ ├── ProtoForwardRangeScan.java │ │ │ ├── PutTest.java │ │ │ └── RangedRowsSetup.java │ │ ├── proto │ │ └── user.proto │ │ └── sbe │ │ └── schema.xml ├── jmh_jmc_rxmulti.sh └── pom.xml ├── .travis.yml ├── rxlmdb-aeron ├── src │ ├── main │ │ └── java │ │ │ ├── org │ │ │ └── deephacks │ │ │ │ └── rxlmdb │ │ │ │ ├── Consts.java │ │ │ │ ├── OpType.java │ │ │ │ ├── ScanPayload.java │ │ │ │ ├── Payloads.java │ │ │ │ ├── KeyValuePayload.java │ │ │ │ ├── RxLmdbServer.java │ │ │ │ ├── RxLmdbRequestHandler.java │ │ │ │ └── RxLmdbClient.java │ │ │ └── rx │ │ │ ├── internal │ │ │ └── reactivestreams │ │ │ │ ├── SubscriberAdapter.java │ │ │ │ ├── RxJavaSynchronizedProducer.java │ │ │ │ └── PublisherAdapter.java │ │ │ └── RxReactiveStreams.java │ └── test │ │ └── java │ │ └── org │ │ └── deephacks │ │ └── rxlmdb │ │ ├── ConcurrencyTest.java │ │ ├── Fixture.java │ │ ├── ConnectivityTest.java │ │ ├── Base.java │ │ └── CrudTest.java └── pom.xml ├── .gitignore ├── rxlmdb ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── deephacks │ │ │ └── rxlmdb │ │ │ ├── Const.java │ │ │ ├── CursorScanner.java │ │ │ ├── Loggable.java │ │ │ ├── DirectMapper.java │ │ │ ├── Streams.java │ │ │ ├── RxTx.java │ │ │ ├── IoUtil.java │ │ │ ├── RxObservables.java │ │ │ ├── KeyValue.java │ │ │ ├── KeyRange.java │ │ │ ├── DirectBufferComparator.java │ │ │ ├── RxLmdb.java │ │ │ ├── Scanners.java │ │ │ └── RxDb.java │ └── test │ │ └── java │ │ └── org │ │ └── deephacks │ │ └── rxlmdb │ │ ├── KeyValueTest.java │ │ ├── EnvTest.java │ │ ├── CursorScannerTest.java │ │ ├── ParallelRangeScanTest.java │ │ ├── DirectBufferComparatorTest.java │ │ ├── BatchTest.java │ │ ├── GetTest.java │ │ ├── Fixture.java │ │ ├── DeleteTest.java │ │ ├── PutTest.java │ │ └── ScanTest.java └── pom.xml ├── rxlmdb-grpc ├── src │ ├── main │ │ ├── proto │ │ │ └── rxdb.proto │ │ └── java │ │ │ └── org │ │ │ └── deephacks │ │ │ └── rxlmdb │ │ │ ├── RxDbGrpcServer.java │ │ │ ├── RxDbServiceGrpc.java │ │ │ └── RxDbGrpcClient.java │ └── test │ │ └── java │ │ └── org │ │ └── deephacks │ │ └── rxlmdb │ │ ├── ConnectivityTest.java │ │ ├── ConcurrencyTest.java │ │ ├── CrudTest.java │ │ └── ScanTest.java └── pom.xml ├── pom.xml └── readme.md /jmh/jmh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | java -jar target/benchmarks.jar $@ 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | 5 | script: "mvn install" 6 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/Country.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | public enum Country { 4 | SE, GB, US 5 | } 6 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/Consts.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | public class Consts { 4 | public static final int DEFAULT_PORT = 39790; 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | *.log 4 | *.iml 5 | *.idea 6 | *.out 7 | .metadata 8 | .settings/ 9 | target/ 10 | eclipse-out/ 11 | test-output/ 12 | bin/ 13 | **/*attach_pid* 14 | *.jfr -------------------------------------------------------------------------------- /jmh/jmh_jmc_rxmulti.sh: -------------------------------------------------------------------------------- 1 | java -jar target/benchmarks.jar org.deephacks.rxlmdb.JMH.rxMulti -jvmArgs="-XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=duration=60s,filename=rxmulti.jfr" 2 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/Const.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | public final class Const { 4 | 5 | public static final boolean TRACING_ENABLED = Boolean.getBoolean("rxlmdb.tracingEnabled"); 6 | 7 | private Const() { 8 | } 9 | } -------------------------------------------------------------------------------- /rxlmdb-aeron/src/test/java/org/deephacks/rxlmdb/ConcurrencyTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.Test; 4 | 5 | public class ConcurrencyTest { 6 | 7 | @Test 8 | public void testMultipleThreadsWithSharedConnection() { 9 | 10 | } 11 | 12 | @Test 13 | public void testMultipleThreadsWithSeparateConnections() { 14 | 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/CursorScanner.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.BufferCursor; 4 | import rx.Subscriber; 5 | 6 | /** 7 | * Provide a BufferCursor for flexible cursor navigation during scans. 8 | */ 9 | public interface CursorScanner { 10 | void execute(BufferCursor cursor, Subscriber subscriber); 11 | } 12 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/test/java/org/deephacks/rxlmdb/Fixture.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | public class Fixture { 4 | 5 | public static KeyValue kv(String prefix, int i) { 6 | byte[] key = String.format("%skey%03d", prefix, i).getBytes(); 7 | byte[] val = String.format("%sval%03d", prefix, i).getBytes(); 8 | return new KeyValue(key, val); 9 | } 10 | 11 | 12 | } 13 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/AddressVal.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.deephacks.vals.Encodable; 4 | import org.deephacks.vals.Id; 5 | import org.deephacks.vals.Val; 6 | 7 | @Val 8 | public interface AddressVal extends Encodable { 9 | @Id(1) byte[] getStreetname(); 10 | @Id(2) int getZipcode(); 11 | @Id(3) int getAreaCode(); 12 | @Id(4) Country getCountry(); 13 | @Id(5) long getTelephone(); 14 | } 15 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/UserVal.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.deephacks.vals.Encodable; 4 | import org.deephacks.vals.Id; 5 | import org.deephacks.vals.Val; 6 | 7 | @Val 8 | public interface UserVal extends Encodable { 9 | @Id(1) byte[] getSsn(); 10 | @Id(2) byte[] getFirstname(); 11 | @Id(3) byte[] getLastname(); 12 | @Id(4) byte[] getEmail(); 13 | @Id(5) long getMobile(); 14 | @Id(6) AddressVal getAddress(); 15 | } 16 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/test/java/org/deephacks/rxlmdb/ConnectivityTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.Test; 4 | 5 | public class ConnectivityTest implements Base { 6 | 7 | @Test 8 | public void testConnectClose() throws Exception { 9 | } 10 | 11 | @Test 12 | public void testMultiConnectClose() { 13 | } 14 | 15 | @Test 16 | public void testClientTimeout() { 17 | } 18 | 19 | @Test 20 | public void testServerTimeout() { 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /jmh/src/main/proto/user.proto: -------------------------------------------------------------------------------- 1 | package generated.proto; 2 | 3 | option java_package = "generated.proto"; 4 | option java_outer_classname = "RxLMDBProtos"; 5 | 6 | message Address { 7 | optional bytes street = 1; 8 | optional uint32 zipcode = 2; 9 | optional uint32 areaCode = 3; 10 | optional uint32 country = 4; 11 | optional uint64 telephone = 5; 12 | } 13 | 14 | message User { 15 | optional bytes ssn = 1; 16 | optional bytes firstname = 2; 17 | optional bytes lastname = 3; 18 | optional bytes email = 4; 19 | optional uint64 mobile = 5; 20 | optional Address address = 6; 21 | } 22 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/OpType.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | enum OpType { 4 | PUT(0x00), 5 | GET(0x01), 6 | DELETE(0x02), 7 | BATCH(0x03), 8 | SCAN(0x04); 9 | 10 | private static OpType[] typesById; 11 | 12 | final int id; 13 | 14 | static { 15 | int max = 0; 16 | 17 | for (OpType t : values()) { 18 | max = Math.max(t.id, max); 19 | } 20 | 21 | typesById = new OpType[max + 1]; 22 | 23 | for (OpType t : values()) { 24 | typesById[t.id] = t; 25 | } 26 | } 27 | 28 | OpType(final int id) 29 | { 30 | this.id = id; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/BigZeroCopyForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.openjdk.jmh.annotations.*; 4 | 5 | import java.util.concurrent.TimeUnit; 6 | 7 | @BenchmarkMode(Mode.Throughput) 8 | @OutputTimeUnit(TimeUnit.SECONDS) 9 | @State(Scope.Thread) 10 | @Measurement(iterations = 5) 11 | @Warmup(iterations = 10) 12 | @Fork(value = 2) 13 | public class BigZeroCopyForwardRangeScan { 14 | static RangedRowsSetup setup = new RangedRowsSetup(BigKeyValueForwardRangeScan.class); 15 | 16 | @State(Scope.Thread) 17 | public static class PlainThread extends AbstractPlainThread { 18 | public PlainThread() { 19 | super(setup, cursor -> cursor.keyByte(0)); 20 | } 21 | } 22 | 23 | @Setup 24 | public void setup() { 25 | setup.writeBigKeyValue(); 26 | } 27 | 28 | @Benchmark 29 | public void plain(PlainThread t) { 30 | t.next(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/Loggable.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | public interface Loggable { 7 | 8 | default void info(String message, Object... args) { 9 | logger().info(message, args); 10 | } 11 | 12 | default void error(String message, Throwable t) { 13 | logger().error(message, t); 14 | } 15 | 16 | default void debug(String message, Object... args) { 17 | logger().debug(message, args); 18 | } 19 | 20 | default void trace(String message, Object... args) { 21 | logger().trace(message, args); 22 | } 23 | 24 | default boolean isTraceEnabled() { 25 | if (Const.TRACING_ENABLED) { 26 | return logger().isTraceEnabled(); 27 | } else { 28 | return false; 29 | } 30 | } 31 | 32 | default Logger logger() { 33 | return LoggerFactory.getLogger(getClass()); 34 | } 35 | } -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/ScanPayload.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.Frame; 4 | import io.reactivesocket.Payload; 5 | import uk.co.real_logic.agrona.MutableDirectBuffer; 6 | 7 | import java.nio.ByteBuffer; 8 | 9 | class ScanPayload implements Payload { 10 | private final ByteBuffer meta; 11 | private final MutableDirectBuffer metaBuf; 12 | private boolean released = false; 13 | 14 | public ScanPayload() { 15 | this.metaBuf = Payloads.POOL.acquireMutableDirectBuffer(4); 16 | this.metaBuf.putInt(0, OpType.SCAN.id); 17 | this.meta = metaBuf.byteBuffer(); 18 | } 19 | 20 | @Override 21 | public ByteBuffer getData() { 22 | return Frame.NULL_BYTEBUFFER; 23 | } 24 | 25 | @Override 26 | public ByteBuffer getMetadata() { 27 | return meta; 28 | } 29 | 30 | public void release() { 31 | if (!released) { 32 | Payloads.POOL.release(metaBuf); 33 | released = true; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/test/java/org/deephacks/rxlmdb/Base.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.slf4j.impl.SimpleLogger; 4 | import rx.functions.Func1; 5 | import uk.co.real_logic.aeron.driver.Configuration; 6 | 7 | import java.util.concurrent.TimeUnit; 8 | 9 | public interface Base { 10 | 11 | default Func1 keyPrefix(String prefix) { 12 | return kv -> new String(kv.key).startsWith(prefix); 13 | } 14 | 15 | static void setDebugMode() { 16 | String hourNs = Long.toString(TimeUnit.HOURS.toNanos(1)); 17 | System.setProperty("reactivesocket.aeron.tracingEnabled", "true"); 18 | System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "TRACE"); 19 | System.setProperty(Configuration.STATUS_MESSAGE_TIMEOUT_PROP_NAME, hourNs); 20 | System.setProperty(Configuration.CLIENT_LIVENESS_TIMEOUT_PROP_NAME, hourNs); 21 | System.setProperty(Configuration.IMAGE_LIVENESS_TIMEOUT_PROP_NAME, hourNs); 22 | System.setProperty(Configuration.PUBLICATION_LINGER_PROP_NAME, hourNs); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/DirectMapper.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | 5 | /** 6 | * Allow for zero-copy using addresses provided by LMDB instead of 7 | * copying data for each operation. Buffer addresses are valid for the 8 | * duration of a transaction. Accessing buffers outside a transaction 9 | * will cause SIGSEGV. 10 | * 11 | * @param The mapped type. 12 | */ 13 | public interface DirectMapper { 14 | /** 15 | * Provide a zero copy key/value pair applied to a given result. 16 | * @return null will be skipped from result 17 | */ 18 | T map(DirectBuffer key, DirectBuffer value); 19 | 20 | class KeyValueMapper implements DirectMapper { 21 | 22 | @Override 23 | public KeyValue map(DirectBuffer key, DirectBuffer value) { 24 | byte[] k = new byte[key.capacity()]; 25 | key.getBytes(0, k); 26 | byte[] v = new byte[value.capacity()]; 27 | value.getBytes(0, v); 28 | return new KeyValue(k, v); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/main/proto/rxdb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_package = "org.deephacks.rxlmdb"; 4 | 5 | message Empty {} 6 | 7 | message PutMsg { 8 | bytes key = 1; 9 | bytes val = 2; 10 | } 11 | 12 | message GetMsg { 13 | bytes key = 1; 14 | } 15 | 16 | message DeleteMsg { 17 | bytes key = 1; 18 | } 19 | 20 | message KeyRangeMsg { 21 | bytes start = 1; 22 | bytes stop = 2; 23 | KeyRangeType type = 3; 24 | } 25 | 26 | enum KeyRangeType { 27 | FORWARD = 0; 28 | FORWARD_START = 1; 29 | FORWARD_STOP = 2; 30 | FOWARD_RANGE = 3; 31 | BACKWARD = 4; 32 | BACKWARD_START = 5; 33 | BACKWARD_STOP = 6; 34 | BACKWARD_RANGE = 7; 35 | } 36 | 37 | message ValueMsg { 38 | bytes val = 1; 39 | } 40 | 41 | message BooleanMsg { 42 | bool value = 1; 43 | } 44 | 45 | message KeyValueMsg { 46 | bytes key = 1; 47 | bytes val = 2; 48 | } 49 | 50 | service DatabaseService { 51 | rpc Put(PutMsg) returns (Empty); 52 | 53 | rpc Batch(stream PutMsg) returns (Empty); 54 | 55 | rpc Get(GetMsg) returns (ValueMsg); 56 | 57 | rpc Delete(DeleteMsg) returns (BooleanMsg); 58 | 59 | rpc Scan(KeyRangeMsg) returns (stream KeyValueMsg); 60 | } 61 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/Streams.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import java.util.LinkedList; 17 | import java.util.Spliterator; 18 | import java.util.Spliterators; 19 | import java.util.stream.Collectors; 20 | import java.util.stream.StreamSupport; 21 | 22 | class Streams { 23 | 24 | public static LinkedList reverse(LinkedList list) { 25 | return StreamSupport.stream( 26 | Spliterators.spliteratorUnknownSize(list.descendingIterator(), Spliterator.ORDERED), false) 27 | .collect(Collectors.toCollection(LinkedList::new)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/KeyValueForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.openjdk.jmh.annotations.*; 4 | 5 | import java.util.concurrent.TimeUnit; 6 | 7 | @BenchmarkMode(Mode.Throughput) 8 | @OutputTimeUnit(TimeUnit.SECONDS) 9 | @State(Scope.Thread) 10 | @Measurement(iterations = 5) 11 | @Warmup(iterations = 10) 12 | @Fork(value = 2) 13 | public class KeyValueForwardRangeScan { 14 | 15 | static RangedRowsSetup setup = new RangedRowsSetup(KeyValueForwardRangeScan.class); 16 | 17 | @State(Scope.Thread) 18 | public static class RxThread extends AbstractRxThread { 19 | public RxThread() { 20 | super(setup, new DirectMapper.KeyValueMapper()); 21 | } 22 | } 23 | 24 | @State(Scope.Thread) 25 | public static class PlainThread extends AbstractPlainThread { 26 | public PlainThread() { 27 | super(setup, cursor -> new KeyValue(cursor.keyBytes(), cursor.valBytes())); 28 | } 29 | } 30 | 31 | @Setup 32 | public void setup() { 33 | setup.writeSmallKeyValue(); 34 | } 35 | 36 | @Benchmark 37 | public void rx(RxThread t) { 38 | t.next(); 39 | } 40 | 41 | @Benchmark 42 | public void plain(PlainThread t) { 43 | t.next(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/BigKeyValueForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.openjdk.jmh.annotations.*; 4 | 5 | import java.util.concurrent.TimeUnit; 6 | 7 | @BenchmarkMode(Mode.Throughput) 8 | @OutputTimeUnit(TimeUnit.SECONDS) 9 | @State(Scope.Thread) 10 | @Measurement(iterations = 5) 11 | @Warmup(iterations = 10) 12 | @Fork(value = 2) 13 | public class BigKeyValueForwardRangeScan { 14 | 15 | static RangedRowsSetup setup = new RangedRowsSetup(BigKeyValueForwardRangeScan.class); 16 | 17 | @State(Scope.Thread) 18 | public static class RxThread extends AbstractRxThread { 19 | public RxThread() { 20 | super(setup, new DirectMapper.KeyValueMapper()); 21 | } 22 | } 23 | 24 | @State(Scope.Thread) 25 | public static class PlainThread extends AbstractPlainThread { 26 | public PlainThread() { 27 | super(setup, cursor -> new KeyValue(cursor.keyBytes(), cursor.valBytes())); 28 | } 29 | } 30 | 31 | @Setup 32 | public void setup() { 33 | setup.writeBigKeyValue(); 34 | } 35 | 36 | @Benchmark 37 | public void rx(RxThread t) { 38 | t.next(); 39 | } 40 | 41 | @Benchmark 42 | public void plain(PlainThread t) { 43 | t.next(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/AbstractRxThread.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | 4 | import java.util.Collections; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | 9 | public abstract class AbstractRxThread { 10 | static AtomicInteger THREAD_ID = new AtomicInteger(0); 11 | public final int id = THREAD_ID.getAndIncrement(); 12 | public RxTx tx; 13 | public Iterator values; 14 | public Iterator> obs; 15 | public DirectMapper mapper; 16 | private RangedRowsSetup setup; 17 | 18 | public AbstractRxThread(RangedRowsSetup setup, DirectMapper mapper) { 19 | this.tx = setup.lmdb.readTx(); 20 | this.mapper = mapper; 21 | this.setup = setup; 22 | this.values = Collections.emptyIterator(); 23 | this.obs = Collections.emptyIterator(); 24 | } 25 | 26 | public final void next() { 27 | if (values.hasNext()) { 28 | values.next(); 29 | } else if (obs.hasNext()) { 30 | values = obs.next().iterator(); 31 | } else { 32 | obs = setup.db.scan(tx, mapper, setup.keyRanges[id]) 33 | .toBlocking().toIterable().iterator(); 34 | values = obs.next().iterator(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/RxTx.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | 17 | import org.fusesource.lmdbjni.Transaction; 18 | 19 | import java.io.Closeable; 20 | 21 | public class RxTx implements Closeable { 22 | final Transaction tx; 23 | /** user managed transactions are closed by user */ 24 | final boolean isUserManaged; 25 | 26 | RxTx(Transaction tx, boolean isUserManaged) { 27 | this.tx = tx; 28 | this.isUserManaged = isUserManaged; 29 | } 30 | 31 | public void abort() { 32 | tx.abort(); 33 | } 34 | 35 | public void commit() { 36 | tx.commit(); 37 | } 38 | 39 | public void close() { 40 | tx.close(); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/KeyValueTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | import org.junit.Test; 5 | 6 | import java.nio.ByteBuffer; 7 | import java.nio.ByteOrder; 8 | 9 | import static com.google.common.truth.Truth.assertThat; 10 | 11 | public class KeyValueTest { 12 | 13 | @Test 14 | public void testFromDirectBuffer() { 15 | ByteOrder order = ByteOrder.BIG_ENDIAN; 16 | DirectBuffer buffer = new DirectBuffer(ByteBuffer.allocateDirect(1 + 2 + 4 + 8)); 17 | buffer.putByte(0, (byte) 1); 18 | buffer.putShort(1, (short) 2, order); 19 | buffer.putInt(3, 3, order); 20 | buffer.putLong(7, 4, order); 21 | 22 | KeyValue kv = new KeyValue(buffer, buffer); 23 | DirectBuffer k = kv.keyBuffer(); 24 | DirectBuffer v = kv.valueBuffer(); 25 | 26 | assertThat(k.getByte(0)).isEqualTo((byte) 1); 27 | assertThat(v.getByte(0)).isEqualTo((byte) 1); 28 | 29 | assertThat(k.getShort(1, order)).isEqualTo((short) 2); 30 | assertThat(v.getShort(1, order)).isEqualTo((short) 2); 31 | 32 | assertThat(k.getInt(3, order)).isEqualTo(3); 33 | assertThat(v.getInt(3, order)).isEqualTo(3); 34 | 35 | assertThat(k.getLong(7, order)).isEqualTo(4L); 36 | assertThat(v.getLong(7, order)).isEqualTo(4L); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/Payloads.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.Frame; 4 | import io.reactivesocket.Payload; 5 | import io.reactivesocket.internal.frame.ThreadLocalFramePool; 6 | 7 | import java.nio.ByteBuffer; 8 | 9 | class Payloads { 10 | static final ThreadLocalFramePool POOL = new ThreadLocalFramePool(); 11 | 12 | public static final Payload EMPTY_PAYLOAD = new Payload() { 13 | @Override 14 | public ByteBuffer getData() { 15 | return Frame.NULL_BYTEBUFFER; 16 | } 17 | 18 | @Override 19 | public ByteBuffer getMetadata() { 20 | return Frame.NULL_BYTEBUFFER; 21 | } 22 | }; 23 | 24 | public static Payload create(final ByteBuffer data, ByteBuffer meta) { 25 | 26 | return new Payload() { 27 | public ByteBuffer getData() { 28 | return data; 29 | } 30 | 31 | @Override 32 | public ByteBuffer getMetadata() { 33 | return meta; 34 | } 35 | }; 36 | } 37 | 38 | public static Payload create(final ByteBuffer data) { 39 | return new Payload() { 40 | public ByteBuffer getData() { 41 | return data; 42 | } 43 | 44 | @Override 45 | public ByteBuffer getMetadata() { 46 | return Frame.NULL_BYTEBUFFER; 47 | } 48 | }; 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/AbstractPlainThread.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.BufferCursor; 4 | import org.fusesource.lmdbjni.DirectBuffer; 5 | import org.fusesource.lmdbjni.Transaction; 6 | 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import java.util.function.Consumer; 9 | 10 | import static org.deephacks.rxlmdb.DirectBufferComparator.compareTo; 11 | 12 | public class AbstractPlainThread { 13 | static AtomicInteger THREAD_ID = new AtomicInteger(0); 14 | public final int id = THREAD_ID.getAndIncrement(); 15 | public BufferCursor cursor; 16 | public Transaction tx; 17 | public DirectBuffer stop; 18 | public byte[] start; 19 | public Consumer consumer; 20 | 21 | public AbstractPlainThread(RangedRowsSetup setup, Consumer consumer) { 22 | this.tx = setup.lmdb.env.createReadTransaction(); 23 | this.cursor = setup.db.db.bufferCursor(tx); 24 | this.stop = new DirectBuffer(setup.keyRanges[id].stop); 25 | this.start = setup.keyRanges[id].start; 26 | this.cursor.seek(start); 27 | this.consumer = consumer; 28 | } 29 | 30 | public final void next() { 31 | if (cursor.next() && compareTo(cursor.keyBuffer(), stop) <= 0) { 32 | consumer.accept(cursor); 33 | } else { 34 | cursor.seek(start); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /jmh/src/main/sbe/schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/ValsForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | import org.openjdk.jmh.annotations.*; 5 | 6 | import java.util.concurrent.TimeUnit; 7 | 8 | @BenchmarkMode(Mode.Throughput) 9 | @OutputTimeUnit(TimeUnit.SECONDS) 10 | @State(Scope.Thread) 11 | @Measurement(iterations = 5) 12 | @Warmup(iterations = 10) 13 | @Fork(value = 2) 14 | public class ValsForwardRangeScan { 15 | 16 | static RangedRowsSetup setup = new RangedRowsSetup(ValsForwardRangeScan.class); 17 | 18 | @State(Scope.Thread) 19 | public static class PlainThread extends AbstractPlainThread { 20 | public PlainThread() { 21 | super(setup, cursor -> parseFrom(cursor.valBuffer()).getSsn()); 22 | } 23 | } 24 | 25 | @State(Scope.Thread) 26 | public static class RxThread extends AbstractRxThread { 27 | public RxThread() { 28 | super(setup, (key, value) -> parseFrom(value).getSsn()); 29 | } 30 | } 31 | 32 | @Setup 33 | public void setup() { 34 | setup.writeValsRanges(); 35 | } 36 | 37 | @Benchmark 38 | public void rx(RxThread t) { 39 | t.next(); 40 | } 41 | 42 | @Benchmark 43 | public void plain(PlainThread t) { 44 | t.next(); 45 | } 46 | 47 | static final UserVal parseFrom(DirectBuffer value) { 48 | org.deephacks.vals.DirectBuffer buffer = 49 | new org.deephacks.vals.DirectBuffer(value.addressOffset(), value.capacity()); 50 | return UserValBuilder.parseFrom(buffer); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/EnvTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import java.nio.file.Path; 8 | import java.util.UUID; 9 | 10 | import static com.google.common.truth.Truth.assertThat; 11 | 12 | public class EnvTest { 13 | RxDb db; 14 | 15 | @Before 16 | public void before() { 17 | db = RxDb.tmp(); 18 | } 19 | 20 | @After 21 | public void after() { 22 | db.close(); 23 | db.lmdb.close(); 24 | } 25 | 26 | @Test 27 | public void testTmpPath() { 28 | Path path = db.lmdb.getPath(); 29 | assertThat(path.toString()).startsWith(IoUtil.TMP_DIR); 30 | } 31 | 32 | @Test 33 | public void testSize() { 34 | long size = db.lmdb.getSize(); 35 | assertThat(size).isGreaterThan(64_000_000L); 36 | assertThat(size).isLessThan(128_000_000L); 37 | } 38 | 39 | @Test 40 | public void testPath() { 41 | String tmp = IoUtil.TMP_DIR + UUID.randomUUID().toString(); 42 | RxLmdb lmdb = RxLmdb.builder().path(tmp).build(); 43 | Path path = lmdb.getPath(); 44 | assertThat(path.toString()).isEqualTo(tmp); 45 | } 46 | 47 | @Test 48 | public void tesAllEnv() { 49 | String tmp = IoUtil.TMP_DIR + UUID.randomUUID().toString(); 50 | RxLmdb.builder() 51 | .path(tmp) 52 | .fixedmap() 53 | .mapAsync() 54 | .noLock() 55 | .noMemInit() 56 | .noMetaSync() 57 | .noReadahead() 58 | .noSync() 59 | .build(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/IoUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | import java.nio.file.Files; 19 | import java.nio.file.Path; 20 | import java.nio.file.Paths; 21 | import java.util.Optional; 22 | 23 | class IoUtil { 24 | static final String TMP_DIR = System.getProperty("java.io.tmpdir") + File.separator + "rxlmdb"; 25 | 26 | static Path createTmpDir() { 27 | try { 28 | Path path = Paths.get(TMP_DIR); 29 | Files.createDirectories(path); 30 | return Files.createTempDirectory(path, ""); 31 | } catch (IOException e) { 32 | throw new RuntimeException(e); 33 | } 34 | } 35 | 36 | static Path createPathOrTemp(Path path) { 37 | Path aPath = Optional.ofNullable(path).orElse(createTmpDir()); 38 | try { 39 | Files.createDirectories(aPath); 40 | } catch (IOException e) { 41 | throw new IllegalArgumentException(e); 42 | } 43 | return aPath; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/KeyValueForwardSkipRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.openjdk.jmh.annotations.*; 4 | 5 | import java.nio.ByteOrder; 6 | import java.util.concurrent.TimeUnit; 7 | 8 | @BenchmarkMode(Mode.Throughput) 9 | @OutputTimeUnit(TimeUnit.SECONDS) 10 | @State(Scope.Thread) 11 | @Measurement(iterations = 5) 12 | @Warmup(iterations = 10) 13 | @Fork(value = 2) 14 | public class KeyValueForwardSkipRangeScan { 15 | 16 | static RangedRowsSetup setup = new RangedRowsSetup(KeyValueForwardSkipRangeScan.class); 17 | static KeyValue kv = new KeyValue(new byte[0], new byte[0]); 18 | 19 | @State(Scope.Thread) 20 | public static class RxThread extends AbstractRxThread { 21 | public RxThread() { 22 | super(setup, (key, value) -> { 23 | if ((key.getInt(1, ByteOrder.BIG_ENDIAN) & 5) == 0) { 24 | return new KeyValue(key, value); 25 | } 26 | return kv; 27 | }); 28 | } 29 | } 30 | 31 | @State(Scope.Thread) 32 | public static class PlainThread extends AbstractPlainThread { 33 | public PlainThread() { 34 | super(setup, cursor -> { 35 | if ((cursor.keyBuffer().getInt(1, ByteOrder.BIG_ENDIAN) & 5) == 0) { 36 | new KeyValue(cursor.keyBytes(), cursor.valBytes()); 37 | } 38 | }); 39 | } 40 | } 41 | 42 | @Setup 43 | public void setup() { 44 | setup.writeSmallKeyValue(); 45 | } 46 | 47 | @Benchmark 48 | public void rx(RxThread t) { 49 | t.next(); 50 | } 51 | 52 | @Benchmark 53 | public void plain(PlainThread t) { 54 | t.next(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/SbeForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | import org.openjdk.jmh.annotations.*; 5 | 6 | import java.util.concurrent.TimeUnit; 7 | 8 | @BenchmarkMode(Mode.Throughput) 9 | @OutputTimeUnit(TimeUnit.SECONDS) 10 | @State(Scope.Thread) 11 | @Measurement(iterations = 5) 12 | @Warmup(iterations = 10) 13 | @Fork(value = 2) 14 | public class SbeForwardRangeScan { 15 | 16 | static RangedRowsSetup setup = new RangedRowsSetup(SbeForwardRangeScan.class); 17 | 18 | @State(Scope.Thread) 19 | public static class PlainThread extends AbstractPlainThread { 20 | public PlainThread() { 21 | super(setup, cursor -> parseFrom(cursor.valBuffer())); 22 | } 23 | } 24 | 25 | @State(Scope.Thread) 26 | public static class RxThread extends AbstractRxThread { 27 | public RxThread() { 28 | super(setup, (key, value) -> parseFrom(value)); 29 | } 30 | } 31 | 32 | @Setup 33 | public void setup() { 34 | setup.writeSbeRanges(); 35 | } 36 | 37 | @Benchmark 38 | public void rx(RxThread t) { 39 | t.next(); 40 | } 41 | 42 | @Benchmark 43 | public void plain(PlainThread t) { 44 | t.next(); 45 | } 46 | 47 | static final generated.sbe.User parseFrom(DirectBuffer value) { 48 | uk.co.real_logic.sbe.codec.java.DirectBuffer buffer = 49 | new uk.co.real_logic.sbe.codec.java.DirectBuffer(value.addressOffset(), value.capacity()); 50 | generated.sbe.User user = new generated.sbe.User(); 51 | user.wrapForDecode(buffer, 0, 0, 0); 52 | byte[] bytes = new byte[16]; 53 | user.getEmail(bytes, 0, 16); 54 | return user; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/CursorScannerTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | import static com.google.common.truth.Truth.assertThat; 13 | import static org.deephacks.rxlmdb.Fixture.*; 14 | import static org.deephacks.rxlmdb.RxObservables.toStreamBlocking; 15 | 16 | public class CursorScannerTest { 17 | RxDb db; 18 | 19 | @Before 20 | public void before() { 21 | db = RxDb.tmp(); 22 | db.put(Observable.from(_1_to_9)); 23 | } 24 | 25 | @After 26 | public void after() { 27 | db.close(); 28 | db.lmdb.close(); 29 | } 30 | 31 | @Test 32 | public void testScanSingle() { 33 | LinkedList expected = Fixture.range(__1, __1); 34 | toStreamBlocking(db.cursor((cursor, subscriber) -> { 35 | cursor.first(); 36 | subscriber.onNext(cursor.keyBytes()); 37 | })).forEach(key -> assertThat(expected.pollFirst().key()).isEqualTo(key)); 38 | assertThat(expected).isEqualTo(new LinkedList<>()); 39 | } 40 | 41 | @Test 42 | public void testScanMultiple() { 43 | List list = toStreamBlocking(db.cursor((cursor, subscriber) -> { 44 | cursor.first(); 45 | subscriber.onNext(cursor.keyBytes()); 46 | cursor.last(); 47 | subscriber.onNext(cursor.keyBytes()); 48 | })).collect(Collectors.toList()); 49 | assertThat(list.size()).isEqualTo(2); 50 | assertThat((byte[]) list.get(0)).isEqualTo(__1); 51 | assertThat((byte[]) list.get(1)).isEqualTo(__9); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/RxObservables.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import rx.Observable; 17 | import rx.exceptions.Exceptions; 18 | 19 | import java.util.List; 20 | import java.util.Optional; 21 | import java.util.function.BiFunction; 22 | import java.util.stream.Stream; 23 | import java.util.stream.StreamSupport; 24 | 25 | class RxObservables { 26 | static Stream toStreamBlocking(Observable> observable) { 27 | Stream> stream = StreamSupport.stream(observable 28 | .doOnError(Exceptions::propagate) 29 | .toBlocking().toIterable().spliterator(), false); 30 | Stream reduce = stream.map(list -> list.spliterator()) 31 | .map(split -> StreamSupport.stream(split, false)) 32 | .reduce(Stream.empty(), (s1, s2) -> Stream.concat(s1, s2)); 33 | return reduce; 34 | } 35 | 36 | static Stream toSingleStreamBlocking(Observable observable) { 37 | return StreamSupport.stream(observable 38 | .doOnError(Exceptions::propagate) 39 | .toBlocking().toIterable().spliterator(), false); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/ProtoForwardRangeScan.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import com.squareup.wire.Wire; 4 | import generated.proto.User; 5 | import org.fusesource.lmdbjni.DirectBuffer; 6 | import org.openjdk.jmh.annotations.*; 7 | 8 | import java.io.IOException; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | 12 | @BenchmarkMode(Mode.Throughput) 13 | @OutputTimeUnit(TimeUnit.SECONDS) 14 | @State(Scope.Thread) 15 | @Measurement(iterations = 5) 16 | @Warmup(iterations = 10) 17 | @Fork(value = 2) 18 | public class ProtoForwardRangeScan { 19 | 20 | static RangedRowsSetup setup = new RangedRowsSetup(ProtoForwardRangeScan.class); 21 | static Wire wire = new Wire(); 22 | 23 | @State(Scope.Thread) 24 | public static class PlainThread extends AbstractPlainThread { 25 | public PlainThread() { 26 | super(setup, cursor -> parseFrom(cursor.valBuffer()).ssn.toByteArray()); 27 | } 28 | } 29 | 30 | @State(Scope.Thread) 31 | public static class RxThread extends AbstractRxThread { 32 | public RxThread() { 33 | super(setup, (key, value) -> parseFrom(value).ssn.toByteArray()); 34 | } 35 | } 36 | 37 | @Setup 38 | public void setup() { 39 | setup.writeProto(); 40 | } 41 | 42 | @Benchmark 43 | public void rx(RxThread t) { 44 | t.next(); 45 | } 46 | 47 | @Benchmark 48 | public void plain(PlainThread t) { 49 | t.next(); 50 | } 51 | 52 | static final User parseFrom(DirectBuffer value) { 53 | try { 54 | byte[] bytes = new byte[value.capacity()]; 55 | value.getBytes(0, bytes); 56 | return wire.parseFrom(bytes, User.class); 57 | } catch (IOException e) { 58 | throw new RuntimeException(e); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rxlmdb/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | org.deephacks.rxlmdb 6 | rxlmdb-project 7 | 0.0.5-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | rxlmdb 12 | rxlmdb 13 | jar 14 | 15 | 16 | 17 | org.deephacks.lmdbjni 18 | lmdbjni 19 | ${lmdbjni-version} 20 | 21 | 22 | io.reactivex 23 | rxjava 24 | 1.1.1 25 | 26 | 27 | org.slf4j 28 | slf4j-api 29 | 1.7.12 30 | 31 | 32 | org.slf4j 33 | slf4j-simple 34 | 1.7.12 35 | test 36 | 37 | 38 | 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-jar-plugin 44 | 2.6 45 | 46 | 47 | 48 | test-jar 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/test/java/org/deephacks/rxlmdb/ConnectivityTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.grpc.Status; 4 | import io.grpc.StatusRuntimeException; 5 | import org.junit.Test; 6 | 7 | import java.nio.file.Path; 8 | 9 | import static org.hamcrest.CoreMatchers.is; 10 | import static org.junit.Assert.assertArrayEquals; 11 | import static org.junit.Assert.assertThat; 12 | import static org.junit.Assert.fail; 13 | 14 | public class ConnectivityTest { 15 | 16 | @Test 17 | public void testConnectClose() throws Exception { 18 | for (int i = 0; i < 9; i++) { 19 | RxDbGrpcServer server = RxDbGrpcServer.builder().build(); 20 | RxDbGrpcClient client = RxDbGrpcClient.builder().build(); 21 | KeyValue put = Fixture.values[i]; 22 | client.put(put).toBlocking().first(); 23 | KeyValue get = client.get(put.key()).toBlocking().first(); 24 | assertArrayEquals(put.key(), get.key()); 25 | assertArrayEquals(put.value(), get.value()); 26 | client.close(); 27 | server.close(); 28 | } 29 | } 30 | 31 | @Test 32 | public void testServerCloseThenReconnectClient() throws Exception { 33 | RxLmdb lmdb = RxLmdb.tmp(); 34 | Path path = lmdb.getPath(); 35 | RxDbGrpcServer server = RxDbGrpcServer.builder().lmdb(lmdb).build(); 36 | RxDbGrpcClient client = RxDbGrpcClient.builder().build(); 37 | KeyValue kv = Fixture.values[0]; 38 | try { 39 | client.put(kv).toBlocking().first(); 40 | server.close(); 41 | client.get(kv.key()).toBlocking().first(); 42 | fail("should throw"); 43 | } catch (StatusRuntimeException e) { 44 | assertThat(e.getStatus().getCode(), is(Status.UNAVAILABLE.getCode())); 45 | } 46 | lmdb = RxLmdb.builder().path(path).build(); 47 | server = RxDbGrpcServer.builder().lmdb(lmdb).build(); 48 | KeyValue result = client.get(kv.key()).toBlocking().first(); 49 | assertArrayEquals(kv.key(), result.key()); 50 | assertArrayEquals(kv.value(), result.value()); 51 | server.close(); 52 | } 53 | 54 | @Test 55 | public void testClientTimeout() { 56 | } 57 | 58 | @Test 59 | public void testServerTimeout() { 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/ParallelRangeScanTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | 11 | import static com.google.common.truth.Truth.assertThat; 12 | import static org.deephacks.rxlmdb.Fixture.*; 13 | import static org.junit.Assert.assertTrue; 14 | 15 | public class ParallelRangeScanTest { 16 | RxDb db; 17 | 18 | @Before 19 | public void before() { 20 | db = RxDb.tmp(); 21 | db.put(Observable.from(_1_to_9)); 22 | } 23 | 24 | @After 25 | public void after() { 26 | db.close(); 27 | db.lmdb.close(); 28 | } 29 | 30 | @Test(expected = IllegalArgumentException.class) 31 | public void testParallelDifferentTx() { 32 | LinkedList expected = Fixture.range(__2, __5); 33 | Observable> result = db.scan(KeyRange.range(__2, __3), KeyRange.range(__4, __5)); 34 | RxObservables.toStreamBlocking(result) 35 | .map(kv -> kv.key()) 36 | .sorted(DirectBufferComparator.byteArrayComparator()) 37 | .forEach(key -> assertThat(expected.pollFirst().key()).isEqualTo(key)); 38 | assertTrue(expected.isEmpty()); 39 | } 40 | 41 | @Test 42 | public void testParallelSameTx() { 43 | LinkedList expected = Fixture.range(__2, __5); 44 | RxTx tx = db.lmdb.readTx(); 45 | Observable> result = db.scan(tx, KeyRange.range(__2, __3), KeyRange.range(__4, __5)); 46 | RxObservables.toStreamBlocking(result) 47 | .map(kv -> kv.key()) 48 | .sorted(DirectBufferComparator.byteArrayComparator()) 49 | .forEach(key -> assertThat(expected.pollFirst().key()).isEqualTo(key)); 50 | assertTrue(expected.isEmpty()); 51 | } 52 | 53 | @Test 54 | public void testParallelScanMapper() { 55 | DirectMapper scan = (key, value) -> key.getByte(0); 56 | LinkedList expected = Fixture.range(__2, __5); 57 | RxTx tx = db.lmdb.readTx(); 58 | Observable> result = db.scan(tx, scan, KeyRange.range(__2, __3), KeyRange.range(__4, __5)); 59 | 60 | RxObservables.toStreamBlocking(result).sorted() 61 | .forEach(key -> assertThat(expected.pollFirst().key()[0]).isEqualTo(key)); 62 | assertTrue(expected.isEmpty()); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/rx/internal/reactivestreams/SubscriberAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package rx.internal.reactivestreams; 17 | 18 | import org.reactivestreams.Subscriber; 19 | import org.reactivestreams.Subscription; 20 | 21 | import java.util.concurrent.atomic.AtomicBoolean; 22 | 23 | public class SubscriberAdapter implements Subscriber { 24 | 25 | private final rx.Subscriber rxSubscriber; 26 | private final AtomicBoolean started = new AtomicBoolean(); 27 | 28 | public SubscriberAdapter(rx.Subscriber rxSubscriber) { 29 | this.rxSubscriber = rxSubscriber; 30 | } 31 | 32 | @Override 33 | public void onSubscribe(final Subscription rsSubscription) { 34 | if (rsSubscription == null) { 35 | throw new NullPointerException("onSubscribe(null)"); 36 | } 37 | 38 | if (started.compareAndSet(false, true)) { 39 | RxJavaSynchronizedProducer sp = new RxJavaSynchronizedProducer(rsSubscription); 40 | rxSubscriber.add(sp); 41 | rxSubscriber.onStart(); 42 | rxSubscriber.setProducer(sp); 43 | } else { 44 | rsSubscription.cancel(); 45 | } 46 | } 47 | 48 | @Override 49 | public void onNext(T t) { 50 | if (t == null) { 51 | throw new NullPointerException("onNext(null)"); 52 | } 53 | rxSubscriber.onNext(t); 54 | } 55 | 56 | @Override 57 | public void onError(Throwable t) { 58 | if (t == null) { 59 | throw new NullPointerException("onError(null)"); 60 | } 61 | rxSubscriber.onError(t); 62 | } 63 | 64 | @Override 65 | public void onComplete() { 66 | rxSubscriber.onCompleted(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/test/java/org/deephacks/rxlmdb/ConcurrencyTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import java.io.IOException; 8 | import java.util.concurrent.CountDownLatch; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | 13 | import static org.hamcrest.core.Is.is; 14 | import static org.junit.Assert.assertThat; 15 | 16 | public class ConcurrencyTest { 17 | RxDbGrpcServer server; 18 | RxDbGrpcClient client = RxDbGrpcClient.builder().build(); 19 | ExecutorService service; 20 | 21 | @Before 22 | public void before() throws IOException { 23 | server = RxDbGrpcServer.builder().build(); 24 | client = RxDbGrpcClient.builder().build(); 25 | service = Executors.newCachedThreadPool(); 26 | } 27 | 28 | @After 29 | public void after() throws Exception { 30 | client.close(); 31 | server.close(); 32 | service.shutdownNow(); 33 | } 34 | 35 | @Test 36 | public void testMultipleThreadsWithSharedConnection() throws Exception { 37 | CountDownLatch latch = new CountDownLatch(1000); 38 | AtomicInteger value = new AtomicInteger(); 39 | for (int i = 0; i < 1000; i++) { 40 | service.execute(() -> { 41 | int k = value.incrementAndGet(); 42 | client.put(Fixture.kv(k, k)).toBlocking().first(); 43 | latch.countDown(); 44 | }); 45 | } 46 | latch.await(); 47 | Integer count = client.scan().count().toBlocking().first(); 48 | assertThat(count, is(1000)); 49 | } 50 | 51 | @Test 52 | public void testMultipleThreadsWithSeparateConnections() throws Exception { 53 | RxDbGrpcClient client2 = RxDbGrpcClient.builder().build(); 54 | CountDownLatch latch = new CountDownLatch(1000); 55 | AtomicInteger value = new AtomicInteger(); 56 | for (int i = 0; i < 1000; i++) { 57 | service.execute(() -> { 58 | int k = value.incrementAndGet(); 59 | RxDbGrpcClient flipFlop = k % 2 == 0 ? client : client2; 60 | flipFlop.put(Fixture.kv(k, k)).toBlocking().first(); 61 | latch.countDown(); 62 | }); 63 | } 64 | latch.await(); 65 | Integer count = client.scan().count().toBlocking().first(); 66 | assertThat(count, is(1000)); 67 | client2.close(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/DirectBufferComparatorTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | import org.junit.Test; 5 | 6 | import java.util.Arrays; 7 | 8 | import static org.hamcrest.core.Is.is; 9 | import static org.junit.Assert.assertThat; 10 | 11 | public class DirectBufferComparatorTest { 12 | @Test 13 | public void testLessThan8bytes() { 14 | assertThat(compare(key(0, 0), stop()), is(0)); 15 | assertThat(compare(key(0, 0), stop(0, 0)), is(0)); 16 | assertThat(compare(key(0, 0), stop(0 )), is(0)); 17 | assertThat(compare(key(0), stop(0, 0)), is(-1)); 18 | assertThat(compare(key(0), stop(1, 0)), is(-1)); 19 | assertThat(compare(key(0), stop(1 )), is(-1)); 20 | assertThat(compare(key(1, 0), stop(0)), is(1)); 21 | assertThat(compare(key(0, 1), stop(1 )), is(-1)); 22 | } 23 | 24 | @Test 25 | public void testMoreThan8bytes() { 26 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(0,0,0,0,0,0,0,0)), is(0)); 27 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(1,0,0,0,0,0,0,0)), is(-1)); 28 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(0,0,0,0,0,0,0)), is(0)); 29 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(0,1,0,0,0,0,0,0)), is(-1)); 30 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(0,0,0,0,0,0,0,0)), is(0)); 31 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(0,0,0,0,0,0,0,0)), is(0)); 32 | assertThat(compare(key(1,0,0,0,0,0,0,0), stop(1,0,0,0,0,0,0)), is(0)); 33 | assertThat(compare(key(0,0,0,0,0,0,0,0), stop(1,0,0,0,0,0,0)), is(-1)); 34 | assertThat(compare(key(0,1,0,0,0,0,0,0), stop(0,0,0,0,0,0,0,0)), is(1)); 35 | } 36 | 37 | DirectBuffer key(int... bytes) { 38 | return b(bytes); 39 | } 40 | 41 | DirectBuffer stop(int... bytes) { 42 | return b(bytes); 43 | } 44 | 45 | DirectBuffer b(int... bytes) { 46 | byte[] array = new byte[bytes.length]; 47 | for (int i = 0; i < bytes.length; i++) { 48 | array[i] = (byte) bytes[i]; 49 | } 50 | return new DirectBuffer(array); 51 | } 52 | 53 | public static int compare(DirectBuffer key, DirectBuffer stop) { 54 | int result = DirectBufferComparator.compareTo(key, stop); 55 | String debug = String.format("%s %s %s", 56 | Arrays.toString(key.byteArray()), 57 | Arrays.toString(stop.byteArray()), 58 | result 59 | ); 60 | // System.out.println(debug); 61 | return result; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/rx/RxReactiveStreams.java: -------------------------------------------------------------------------------- 1 | package rx; 2 | 3 | import org.reactivestreams.Publisher; 4 | import rx.internal.reactivestreams.PublisherAdapter; 5 | import rx.internal.reactivestreams.SubscriberAdapter; 6 | 7 | /** 8 | * This type provides static factory methods for converting to and from RxJava types and Reactive Streams types. 9 | *

10 | * The Reactive Streams API provides a common API for interoperability 11 | * between different reactive streaming libraries, of which RxJava is one. 12 | * Using the methods of this class, RxJava can collaborate with other such libraries that also implement the standard. 13 | */ 14 | public abstract class RxReactiveStreams { 15 | 16 | private RxReactiveStreams() { 17 | } 18 | 19 | /** 20 | * Convert a Rx {@link Observable} into a Reactive Streams {@link Publisher}. 21 | *

22 | * Use this method when you have an RxJava observable, that you want to be consumed by another library. 23 | * 24 | * @param observable the {@link Observable} to convert 25 | * @return the converted {@link Publisher} 26 | */ 27 | public static Publisher toPublisher(Observable observable) { 28 | return new PublisherAdapter(observable); 29 | } 30 | 31 | /** 32 | * Convert a Reactive Streams {@link Publisher} into a Rx {@link Observable}. 33 | *

34 | * Use this method when you have a stream from another library, that you want to be consume as an RxJava observable. 35 | * 36 | * @param publisher the {@link Publisher} to convert. 37 | * @return the converted {@link Observable} 38 | */ 39 | public static Observable toObservable(final Publisher publisher) { 40 | return Observable.create(new Observable.OnSubscribe() { 41 | @Override 42 | public void call(final rx.Subscriber rxSubscriber) { 43 | publisher.subscribe(toSubscriber(rxSubscriber)); 44 | } 45 | }); 46 | } 47 | 48 | /** 49 | * Convert an RxJava {@link rx.Subscriber} into a Reactive Streams {@link org.reactivestreams.Subscriber}. 50 | * 51 | * @param rxSubscriber an RxJava subscriber 52 | * @return a Reactive Streams subscriber 53 | */ 54 | public static org.reactivestreams.Subscriber toSubscriber(final rx.Subscriber rxSubscriber) { 55 | return new SubscriberAdapter(rxSubscriber); 56 | } 57 | 58 | } -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/KeyValue.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import org.fusesource.lmdbjni.DirectBuffer; 17 | import org.fusesource.lmdbjni.Entry; 18 | 19 | import java.nio.ByteOrder; 20 | import java.util.Arrays; 21 | 22 | public class KeyValue { 23 | private final byte[] key; 24 | private final byte[] value; 25 | 26 | private final DirectBuffer keyBuffer; 27 | private final DirectBuffer valueBuffer; 28 | 29 | KeyValue(Entry entry) { 30 | if (entry == null) { 31 | throw new NullPointerException(); 32 | } 33 | if (entry.getKey() == null) { 34 | throw new NullPointerException(); 35 | } 36 | if (entry.getValue() == null) { 37 | throw new NullPointerException(); 38 | } 39 | this.key = entry.getKey(); 40 | this.value = entry.getValue(); 41 | this.keyBuffer = null; 42 | this.valueBuffer = null; 43 | } 44 | 45 | public KeyValue(byte[] key, byte[] value) { 46 | if (key == null) { 47 | throw new NullPointerException(); 48 | } 49 | if (value == null) { 50 | throw new NullPointerException(); 51 | } 52 | this.key = key; 53 | this.value = value; 54 | this.valueBuffer = null; 55 | this.keyBuffer = null; 56 | } 57 | 58 | public KeyValue(DirectBuffer key, DirectBuffer value) { 59 | if (key == null) { 60 | throw new NullPointerException(); 61 | } 62 | if (value == null) { 63 | throw new NullPointerException(); 64 | } 65 | this.keyBuffer = key; 66 | this.valueBuffer = value; 67 | this.key = null; 68 | this.value = null; 69 | } 70 | 71 | public byte[] key() { 72 | return key; 73 | } 74 | 75 | public byte[] value() { 76 | return value; 77 | } 78 | 79 | public DirectBuffer keyBuffer() { 80 | return keyBuffer; 81 | } 82 | 83 | public DirectBuffer valueBuffer() { 84 | return valueBuffer; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/KeyRange.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.deephacks.rxlmdb; 16 | 17 | import java.util.Arrays; 18 | 19 | import static org.deephacks.rxlmdb.KeyRange.KeyRangeType.*; 20 | 21 | /** 22 | * Ranges are inclusive and keys are prefix matched. 23 | */ 24 | public class KeyRange { 25 | public final byte[] start; 26 | public final byte[] stop; 27 | public final KeyRangeType type; 28 | 29 | KeyRange(byte[] start, byte[] stop, KeyRangeType type) { 30 | this.start = start; 31 | this.stop = stop; 32 | this.type = type; 33 | } 34 | 35 | public static KeyRange forward() { 36 | return new KeyRange(null, null, FORWARD); 37 | } 38 | 39 | public static KeyRange backward() { 40 | return new KeyRange(null, null, BACKWARD); 41 | } 42 | 43 | public static KeyRange atLeast(byte[] start) { 44 | return new KeyRange(start, null, FORWARD_START); 45 | } 46 | 47 | public static KeyRange atLeastBackward(byte[] start) { 48 | return new KeyRange(start, null, BACKWARD_START); 49 | } 50 | 51 | public static KeyRange atMost(byte[] stop) { 52 | return new KeyRange(null, stop, FORWARD_STOP); 53 | } 54 | 55 | public static KeyRange atMostBackward(byte[] stop) { 56 | return new KeyRange(null, stop, BACKWARD_STOP); 57 | } 58 | 59 | public static KeyRange range(byte[] start, byte[] stop) { 60 | if (DirectBufferComparator.compareTo(start, stop) <= 0) { 61 | return new KeyRange(start, stop, FOWARD_RANGE); 62 | } else { 63 | return new KeyRange(start, stop, BACKWARD_RANGE); 64 | } 65 | } 66 | 67 | enum KeyRangeType { 68 | FORWARD, FORWARD_START, FORWARD_STOP, FOWARD_RANGE, BACKWARD, BACKWARD_START, BACKWARD_STOP, BACKWARD_RANGE 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return "KeyRange{" + 74 | "start=" + Arrays.toString(start) + 75 | ", stop=" + Arrays.toString(stop) + 76 | ", type=" + type + 77 | '}'; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /rxlmdb-aeron/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | org.deephacks.rxlmdb 6 | rxlmdb-project 7 | 0.0.4-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | rxlmdb-aeron 12 | rxlmdb-aeron 13 | jar 14 | 15 | 16 | 17 | org.deephacks.rxlmdb 18 | rxlmdb 19 | 0.0.4-SNAPSHOT 20 | 21 | 22 | uk.co.real-logic 23 | aeron-all 24 | ${aeron-version} 25 | 26 | 27 | org.reactivestreams 28 | reactive-streams 29 | ${reactive-streams-version} 30 | 31 | 32 | io.reactivesocket 33 | reactivesocket 34 | ${rxsocket-version} 35 | 36 | 37 | uk.co.real-logic 38 | Agrona 39 | 40 | 41 | 42 | 43 | org.hdrhistogram 44 | HdrHistogram 45 | 2.1.7 46 | 47 | 48 | org.slf4j 49 | slf4j-simple 50 | 1.7.12 51 | test 52 | 53 | 54 | io.reactivesocket 55 | reactivesocket-aeron-core 56 | ${rxsocket-aeron-version} 57 | 58 | 59 | io.reactivesocket 60 | reactivesocket-aeron-client 61 | ${rxsocket-aeron-version} 62 | 63 | 64 | io.reactivesocket 65 | reactivesocket-aeron-server 66 | ${rxsocket-aeron-version} 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/BatchTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | import rx.Subscriber; 8 | import rx.schedulers.Schedulers; 9 | import rx.subjects.PublishSubject; 10 | 11 | import java.util.List; 12 | import java.util.concurrent.TimeUnit; 13 | import java.util.concurrent.atomic.AtomicInteger; 14 | 15 | import static org.hamcrest.core.Is.is; 16 | import static org.junit.Assert.assertThat; 17 | 18 | public class BatchTest { 19 | RxDb db; 20 | RxLmdb lmdb; 21 | 22 | @Before 23 | public void before() { 24 | lmdb = RxLmdb.tmp(); 25 | db = lmdb.dbBuilder().build(); 26 | } 27 | 28 | @After 29 | public void after() { 30 | db.close(); 31 | db.lmdb.close(); 32 | } 33 | 34 | @Test 35 | public void testBatch() throws InterruptedException { 36 | AtomicInteger counter = new AtomicInteger(); 37 | db.batch(burst(counter).buffer(100, TimeUnit.MILLISECONDS, 10)); 38 | Thread.sleep(1000); 39 | List list = db.scan(KeyRange.forward()).toBlocking().first(); 40 | assertThat(list.size(), is(counter.get())); 41 | assertThat(list.get(0).key()[0], is((byte) 1)); 42 | assertThat(list.get(list.size() - 1).key()[0], is((byte) counter.get())); 43 | } 44 | 45 | /** 46 | * An error item should not affect writing other items. 47 | */ 48 | @Test 49 | public void testBatchSingleError() throws InterruptedException { 50 | PublishSubject subject = PublishSubject.create(); 51 | db.batch(subject.observeOn(Schedulers.newThread()).buffer(100, TimeUnit.MILLISECONDS, 10)); 52 | subject.onNext(Fixture.values[0]); 53 | subject.onNext(null); 54 | subject.onNext(Fixture.values[2]); 55 | subject.onCompleted(); 56 | Thread.sleep(500); 57 | List list = db.scan(KeyRange.forward()).toBlocking().first(); 58 | assertThat(list.size(), is(2)); 59 | } 60 | 61 | static Observable burst(AtomicInteger counter) { 62 | return Observable.create((Subscriber s) -> { 63 | int rounds = 5; 64 | while (!s.isUnsubscribed()) { 65 | // burst some number of items 66 | int num = (int) (Math.random() * 10); 67 | for (int i = 0; i < num; i++) { 68 | int j = counter.incrementAndGet(); 69 | s.onNext(new KeyValue(new byte[] { (byte) j }, new byte[] {(byte) j})); 70 | } 71 | try { 72 | // sleep for a random amount of time 73 | Thread.sleep((long) (Math.random() * 100)); 74 | } catch (Exception e) { 75 | // do nothing 76 | } 77 | if (rounds-- == 0) { 78 | s.onCompleted(); 79 | return; 80 | } 81 | } 82 | }).subscribeOn(Schedulers.newThread()); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/KeyValuePayload.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.Payload; 4 | import uk.co.real_logic.agrona.BitUtil; 5 | import uk.co.real_logic.agrona.MutableDirectBuffer; 6 | import uk.co.real_logic.agrona.concurrent.UnsafeBuffer; 7 | 8 | import java.nio.ByteBuffer; 9 | 10 | class KeyValuePayload implements Payload { 11 | private static final byte[] EMPTY = new byte[0]; 12 | private final ByteBuffer data; 13 | private final ByteBuffer meta; 14 | private final MutableDirectBuffer dataBuf; 15 | private final MutableDirectBuffer metaBuf; 16 | 17 | public KeyValuePayload(KeyValue kv, OpType type) { 18 | this(kv.key, kv.value, type); 19 | } 20 | 21 | public KeyValuePayload(byte[] key, OpType type) { 22 | this(key, EMPTY, type); 23 | } 24 | 25 | public KeyValuePayload(byte[] key, byte[] value, OpType type) { 26 | this.dataBuf = Payloads.POOL.acquireMutableDirectBuffer(8 + key.length + value.length); 27 | int offset = 0; 28 | 29 | this.dataBuf.putInt(offset, key.length); 30 | offset += BitUtil.SIZE_OF_INT; 31 | 32 | this.dataBuf.putBytes(offset, key); 33 | offset += key.length; 34 | 35 | this.dataBuf.putInt(offset, value.length); 36 | offset += BitUtil.SIZE_OF_INT; 37 | 38 | this.dataBuf.putBytes(offset, value); 39 | 40 | this.metaBuf = Payloads.POOL.acquireMutableDirectBuffer(4); 41 | this.metaBuf.putInt(0, type.id); 42 | 43 | this.data = dataBuf.byteBuffer(); 44 | this.meta = metaBuf.byteBuffer(); 45 | } 46 | 47 | public static KeyValue getKeyValue(Payload payload) { 48 | ByteBuffer bb = payload.getData(); 49 | if (bb.capacity() == 0) { 50 | return null; 51 | } 52 | UnsafeBuffer data = new UnsafeBuffer(bb); 53 | int offset = 0; 54 | int size = data.getInt(offset); 55 | offset += BitUtil.SIZE_OF_INT; 56 | 57 | byte[] key = new byte[size]; 58 | data.getBytes(offset, key); 59 | offset += size; 60 | 61 | size = data.getInt(offset); 62 | offset += BitUtil.SIZE_OF_INT; 63 | 64 | byte[] value = new byte[size]; 65 | data.getBytes(offset, value); 66 | return new KeyValue(key, value); 67 | } 68 | 69 | public static byte[] getByteArray(Payload payload) { 70 | ByteBuffer bb = payload.getData(); 71 | if (bb.capacity() == 0) { 72 | return null; 73 | } 74 | UnsafeBuffer data = new UnsafeBuffer(bb); 75 | int size = data.getInt(0); 76 | byte[] key = new byte[size]; 77 | data.getBytes(4, key); 78 | return key; 79 | } 80 | 81 | @Override 82 | public ByteBuffer getData() { 83 | return data; 84 | } 85 | 86 | @Override 87 | public ByteBuffer getMetadata() { 88 | return meta; 89 | } 90 | 91 | public void release() { 92 | Payloads.POOL.release(metaBuf); 93 | Payloads.POOL.release(dataBuf); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/GetTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | 8 | import java.util.List; 9 | import java.util.stream.Collectors; 10 | 11 | import static com.google.common.truth.Truth.assertThat; 12 | import static org.deephacks.rxlmdb.Fixture._1_to_9; 13 | import static org.deephacks.rxlmdb.Fixture.*; 14 | 15 | public class GetTest { 16 | RxDb db; 17 | 18 | @Before 19 | public void before() { 20 | db = RxDb.tmp(); 21 | db.put(Observable.from(_1_to_9)); 22 | } 23 | 24 | @After 25 | public void after() { 26 | db.close(); 27 | db.lmdb.close(); 28 | } 29 | 30 | @Test 31 | public void testGetSingle() { 32 | List result = RxObservables.toSingleStreamBlocking(db.get(Observable.just(__1))) 33 | .collect(Collectors.toList()); 34 | assertThat(result).hasSize(1); 35 | assertThat(result.get(0).key()).isEqualTo(__1); 36 | assertThat(db.get(__1)).isEqualTo(__1); 37 | } 38 | 39 | @Test 40 | public void testGetNull() { 41 | List result = RxObservables.toSingleStreamBlocking( 42 | db.get(Observable.from(new byte[][]{__1, null, __2})) 43 | ).collect(Collectors.toList()); 44 | assertThat(result).hasSize(3); 45 | assertThat(result.get(0).key()).isEqualTo(__1); 46 | assertThat(result.get(0).value()).isEqualTo(__1); 47 | assertThat(result.get(1)).isEqualTo(null); 48 | assertThat(result.get(2).key()).isEqualTo(__2); 49 | assertThat(result.get(2).value()).isEqualTo(__2); 50 | } 51 | 52 | 53 | @Test 54 | public void testGetMultiple() { 55 | List result = RxObservables.toSingleStreamBlocking( 56 | db.get(Observable.from(new byte[][]{__3, __1, __2})) 57 | ).collect(Collectors.toList()); 58 | assertThat(result).hasSize(3); 59 | assertThat(result.get(0).key()).isEqualTo(__3); 60 | assertThat(result.get(1).key()).isEqualTo(__1); 61 | assertThat(result.get(2).key()).isEqualTo(__2); 62 | 63 | } 64 | 65 | @Test 66 | public void testGetNonExisting() { 67 | List result = RxObservables.toSingleStreamBlocking(db.get(Observable.just(new byte[]{123}))) 68 | .collect(Collectors.toList()); 69 | assertThat(result).hasSize(1); 70 | assertThat(result.get(0)).isNull(); 71 | } 72 | 73 | @Test 74 | public void testGetSomeExisting() { 75 | List result = RxObservables.toSingleStreamBlocking( 76 | db.get(Observable.from(new byte[][]{ new byte[]{ 123 }, __9, new byte[]{ 123 }, __8, })) 77 | ).collect(Collectors.toList()); 78 | assertThat(result).hasSize(4); 79 | assertThat(result.get(0)).isNull(); 80 | assertThat(result.get(1).key()).isEqualTo(__9); 81 | assertThat(result.get(1).value()).isEqualTo(__9); 82 | assertThat(result.get(2)).isNull(); 83 | assertThat(result.get(3).key()).isEqualTo(__8); 84 | assertThat(result.get(3).value()).isEqualTo(__8); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/test/java/org/deephacks/rxlmdb/CrudTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.AfterClass; 4 | import org.junit.BeforeClass; 5 | import org.junit.Test; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | import static org.hamcrest.core.Is.is; 11 | import static org.junit.Assert.*; 12 | 13 | public class CrudTest implements Base { 14 | static { 15 | // Base.setDebugMode(); 16 | } 17 | static RxLmdbServer server; 18 | static RxLmdbClient client; 19 | 20 | @BeforeClass 21 | public static void beforeClass() { 22 | server = RxLmdbServer.builder().build(); 23 | client = RxLmdbClient.builder().build().connectAndWait(); 24 | } 25 | 26 | @AfterClass 27 | public static void afterClass() throws Exception { 28 | } 29 | 30 | @Test 31 | public void testPutGetDelete() { 32 | KeyValue kv1 = Fixture.kv("putGetDelete", 1); 33 | assertTrue(client.put(kv1).toBlocking().first()); 34 | KeyValue kv2 = null; 35 | for (int i = 0; i < 100; i++) { 36 | kv2 = client.get(kv1.key).toBlocking().first(); 37 | } 38 | assertArrayEquals(kv1.key, kv2.key); 39 | assertArrayEquals(kv1.value, kv2.value); 40 | 41 | assertTrue(client.delete(kv1.key).toBlocking().first()); 42 | assertNull(client.get(kv1.key).toBlocking().firstOrDefault(null)); 43 | } 44 | 45 | @Test 46 | public void deleteNonExisting() { 47 | assertFalse(client.delete(new byte[] {9,9,9}).toBlocking().first()); 48 | } 49 | 50 | @Test 51 | public void getNonExisting() { 52 | assertNull(client.get(new byte[] {9,9,9}).toBlocking().firstOrDefault(null)); 53 | } 54 | 55 | @Test 56 | public void getEmptyKey() { 57 | assertNull(client.get(new byte[0]).toBlocking().firstOrDefault(null)); 58 | } 59 | 60 | @Test 61 | public void testBatch() throws InterruptedException { 62 | List kvs = new ArrayList<>(); 63 | for (int i = 0; i < 1000; i++) { 64 | KeyValue kv = Fixture.kv("batch", i); 65 | kvs.add(kv); 66 | client.batch(kv); 67 | } 68 | 69 | List list = client.scan().filter(keyPrefix("batch")) 70 | .toList().toBlocking().first(); 71 | assertThat(list.size(), is(1000)); 72 | for (int i = 0; i < 1000; i++) { 73 | KeyValue kv = kvs.get(i); 74 | assertThat(new String(kv.key), is(new String(list.get(i).key))); 75 | } 76 | } 77 | 78 | @Test 79 | public void testScan() throws InterruptedException { 80 | List kvs = new ArrayList<>(); 81 | for (int i = 0; i < 100; i++) { 82 | KeyValue kv = Fixture.kv("scan", i); 83 | kvs.add(kv); 84 | assertTrue(client.put(kv).toBlocking().first()); 85 | } 86 | List list = client.scan().filter(keyPrefix("scan")) 87 | .toList().toBlocking().first(); 88 | assertThat(list.size(), is(100)); 89 | for (int i = 0; i < 100; i++) { 90 | KeyValue kv = kvs.get(i); 91 | assertThat(new String(kv.key), is(new String(list.get(i).key))); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/Fixture.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | import java.util.stream.Stream; 6 | 7 | 8 | public class Fixture { 9 | /** number of underscores signifies repeated number of b */ 10 | 11 | public static final byte[] __1 = new byte[] { 1, 1 }; 12 | public static final byte[] __2 = new byte[] { 2, 2 }; 13 | public static final byte[] __3 = new byte[] { 3, 3 }; 14 | public static final byte[] __4 = new byte[] { 4, 4 }; 15 | public static final byte[] __5 = new byte[] { 5, 5 }; 16 | public static final byte[] __6 = new byte[] { 6, 6 }; 17 | public static final byte[] __7 = new byte[] { 7, 7 }; 18 | public static final byte[] __8 = new byte[] { 8, 8 }; 19 | public static final byte[] __9 = new byte[] { 9, 9 }; 20 | 21 | public static final byte[] _1 = new byte[] { 1 }; 22 | public static final byte[] _2 = new byte[] { 2 }; 23 | public static final byte[] _3 = new byte[] { 3 }; 24 | public static final byte[] _4 = new byte[] { 4 }; 25 | public static final byte[] _5 = new byte[] { 5 }; 26 | public static final byte[] _6 = new byte[] { 6 }; 27 | public static final byte[] _7 = new byte[] { 7 }; 28 | public static final byte[] _8 = new byte[] { 8 }; 29 | public static final byte[] _9 = new byte[] { 9 }; 30 | 31 | public static final byte[][] keys = new byte[][] { __1, __2, __3, __4, __5, __6, __7, __8, __9}; 32 | 33 | public static final KeyValue[] values = new KeyValue[] { 34 | new KeyValue(__1, __1), 35 | new KeyValue(__2, __2), 36 | new KeyValue(__3, __3), 37 | new KeyValue(__4, __4), 38 | new KeyValue(__5, __5), 39 | new KeyValue(__6, __6), 40 | new KeyValue(__7, __7), 41 | new KeyValue(__8, __8), 42 | new KeyValue(__9, __9) 43 | }; 44 | 45 | public static final LinkedList _1_to_9 = Stream.of(values) 46 | .collect(Collectors.toCollection(LinkedList::new)); 47 | 48 | public static final LinkedList _9_to_1 = Streams.reverse(_1_to_9); 49 | 50 | /** 51 | * inclusive ranges 52 | */ 53 | public static LinkedList range(byte[] start, byte[] stop) { 54 | LinkedList result; 55 | if (DirectBufferComparator.compareTo(start, stop) < 0) { 56 | result = _1_to_9.stream() 57 | .filter(b -> DirectBufferComparator.within(b.key(), start, stop)) 58 | .collect(Collectors.toCollection(LinkedList::new)); 59 | 60 | } else { 61 | result = _9_to_1.stream() 62 | .filter(b -> DirectBufferComparator.within(b.key(), stop, start)) 63 | .collect(Collectors.toCollection(LinkedList::new)); 64 | } 65 | if (result.isEmpty()) { 66 | throw new IllegalArgumentException("Range was empty"); 67 | } 68 | return result; 69 | } 70 | 71 | public static KeyValue kv(int key, int val) { 72 | byte[] k = String.format("%03d", key).getBytes(); 73 | byte[] v = String.format("%03d", val).getBytes(); 74 | return new KeyValue(k, v); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/main/java/org/deephacks/rxlmdb/RxDbGrpcServer.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.grpc.ServerInterceptors; 4 | import io.grpc.internal.ServerImpl; 5 | import io.grpc.netty.NettyServerBuilder; 6 | 7 | import java.io.IOException; 8 | import java.util.Optional; 9 | 10 | public class RxDbGrpcServer { 11 | static final int DEFAULT_PORT = 18080; 12 | private final int port; 13 | private final String host; 14 | private RxLmdb lmdb; 15 | private RxDb db; 16 | private ServerImpl server; 17 | 18 | private RxDbGrpcServer(Builder builder) throws IOException { 19 | this.port = Optional.ofNullable(builder.port).orElse(DEFAULT_PORT); 20 | this.host = Optional.ofNullable(builder.host).orElse("localhost"); 21 | this.lmdb = Optional.ofNullable(builder.lmdb).orElseGet(() -> RxLmdb.tmp()); 22 | this.db = Optional.ofNullable(builder.db).orElseGet(() -> RxDbGrpcServer.this.lmdb.dbBuilder().build()); 23 | if (builder.builder != null) { 24 | this.server = builder.builder 25 | .addService(ServerInterceptors.intercept( 26 | DatabaseServiceGrpc.bindService(new RxDbServiceGrpc(this.db)))) 27 | .build().start(); 28 | } else { 29 | this.server = NettyServerBuilder.forPort(port) 30 | .addService(ServerInterceptors.intercept( 31 | DatabaseServiceGrpc.bindService(new RxDbServiceGrpc(this.db)))) 32 | .build().start(); 33 | } 34 | } 35 | 36 | public int getPort() { 37 | return port; 38 | } 39 | 40 | public String getHost() { 41 | return host; 42 | } 43 | 44 | public void close() throws Exception { 45 | runQuitely(() -> db.close()); 46 | runQuitely(() -> lmdb.close()); 47 | runQuitely(() -> server.shutdown()); 48 | server.awaitTermination(); 49 | } 50 | 51 | public static Builder builder() { 52 | return new Builder(); 53 | } 54 | 55 | public static class Builder { 56 | private Integer port; 57 | private String host; 58 | private RxLmdb lmdb; 59 | private RxDb db; 60 | private NettyServerBuilder builder; 61 | 62 | public RxDbGrpcServer build() throws IOException { 63 | return new RxDbGrpcServer(this); 64 | } 65 | 66 | public Builder host(String host) { 67 | this.host = host; 68 | return this; 69 | } 70 | 71 | public Builder port(int port) { 72 | this.port = port; 73 | return this; 74 | } 75 | 76 | public Builder lmdb(RxLmdb lmdb) { 77 | this.lmdb = lmdb; 78 | return this; 79 | } 80 | 81 | public Builder db(RxDb db) { 82 | this.db = db; 83 | return this; 84 | } 85 | 86 | public Builder nettyServerBuilder(NettyServerBuilder builder) { 87 | this.builder = builder; 88 | return this; 89 | } 90 | } 91 | 92 | static void runQuitely(CodeBlock block) { 93 | try { 94 | block.execute(); 95 | } catch (Exception e) { 96 | e.printStackTrace(); 97 | } 98 | } 99 | 100 | interface CodeBlock { 101 | void execute(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/PutTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.ByteUnit; 4 | import org.openjdk.jmh.annotations.*; 5 | import rx.Observable; 6 | import rx.subjects.PublishSubject; 7 | import rx.subjects.SerializedSubject; 8 | 9 | import java.io.IOException; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | import java.nio.file.Paths; 13 | import java.util.concurrent.TimeUnit; 14 | import java.util.concurrent.atomic.AtomicInteger; 15 | 16 | @BenchmarkMode(Mode.Throughput) 17 | @OutputTimeUnit(TimeUnit.SECONDS) 18 | @State(Scope.Thread) 19 | @Measurement(iterations = 5) 20 | @Warmup(iterations = 5) 21 | @Fork(value = 1) 22 | @Threads(value = 12) 23 | public class PutTest { 24 | 25 | static RxDb db; 26 | static RxLmdb lmdb; 27 | static SerializedSubject subject; 28 | 29 | static { 30 | try { 31 | Path path = Paths.get("/tmp/rxlmdb-jmh-BatchTest"); 32 | Files.createDirectories(path); 33 | lmdb = RxLmdb.builder().path(path).size(1, ByteUnit.GIBIBYTES).build(); 34 | db = RxDb.builder().lmdb(lmdb).build(); 35 | } catch (IOException e) { 36 | throw new RuntimeException(e); 37 | } 38 | subject = PublishSubject.create().toSerialized(); 39 | db.batch(subject.buffer(10, TimeUnit.NANOSECONDS, 4096)); 40 | } 41 | 42 | @State(Scope.Thread) 43 | public static class RxThread { 44 | public static final byte[] _1 = new byte[]{1}; 45 | public static final byte[] _2 = new byte[]{2}; 46 | public static final byte[] _3 = new byte[]{3}; 47 | public static final byte[] _4 = new byte[]{4}; 48 | public static final byte[] _5 = new byte[]{5}; 49 | public static final byte[] _6 = new byte[]{6}; 50 | public static final byte[] _7 = new byte[]{7}; 51 | public static final byte[] _8 = new byte[]{8}; 52 | public static final byte[] _9 = new byte[]{9}; 53 | 54 | public static final KeyValue[] values = new KeyValue[]{ 55 | new KeyValue(_1, _1), 56 | new KeyValue(_2, _2), 57 | new KeyValue(_3, _3), 58 | new KeyValue(_4, _4), 59 | new KeyValue(_5, _5), 60 | new KeyValue(_6, _6), 61 | new KeyValue(_7, _7), 62 | new KeyValue(_8, _8), 63 | new KeyValue(_9, _9) 64 | }; 65 | 66 | public static final Observable[] observables = new Observable[]{ 67 | Observable.just(values[0]), 68 | Observable.just(values[1]), 69 | Observable.just(values[2]), 70 | Observable.just(values[3]), 71 | Observable.just(values[4]), 72 | Observable.just(values[5]), 73 | Observable.just(values[6]), 74 | Observable.just(values[7]), 75 | Observable.just(values[8]) 76 | }; 77 | 78 | RxTx tx = null; 79 | AtomicInteger counter = new AtomicInteger(); 80 | 81 | public RxThread() { 82 | } 83 | 84 | public void batch() { 85 | int i = counter.incrementAndGet(); 86 | subject.onNext(values[i % 9]); 87 | } 88 | 89 | public void put() { 90 | if (tx == null) { 91 | tx = lmdb.writeTx(); 92 | } 93 | int num = counter.incrementAndGet() % 9; 94 | db.put(tx, observables[num]); 95 | if (num == 0) { 96 | tx.commit(); 97 | tx = null; 98 | } 99 | } 100 | } 101 | 102 | @Benchmark 103 | public void batch(RxThread t) { 104 | t.batch(); 105 | } 106 | 107 | @Benchmark 108 | public void put(RxThread t) { 109 | t.put(); 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/RxLmdbServer.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.aeron.server.ReactiveSocketAeronServer; 4 | import uk.co.real_logic.aeron.driver.MediaDriver; 5 | import uk.co.real_logic.aeron.driver.ThreadingMode; 6 | import uk.co.real_logic.agrona.concurrent.BackoffIdleStrategy; 7 | import uk.co.real_logic.agrona.concurrent.NoOpIdleStrategy; 8 | 9 | import java.util.Optional; 10 | 11 | public class RxLmdbServer { 12 | static { 13 | System.setProperty("reactivesocket.aeron.clientConcurrency", "1"); 14 | } 15 | 16 | private final int port; 17 | private final String host; 18 | 19 | private ReactiveSocketAeronServer server; 20 | private RxLmdb lmdb; 21 | private RxDb db; 22 | private MediaDriver mediaDriver; 23 | private final boolean manageMediaDriver; 24 | 25 | private RxLmdbServer(Builder builder) { 26 | this.port = Optional.ofNullable(builder.port).orElse(Consts.DEFAULT_PORT); 27 | this.host = Optional.ofNullable(builder.host).orElse("localhost"); 28 | this.lmdb = Optional.ofNullable(builder.lmdb).orElseGet(() -> RxLmdb.tmp()); 29 | this.db = Optional.ofNullable(builder.db).orElseGet(() -> RxLmdbServer.this.lmdb.dbBuilder().build()); 30 | this.manageMediaDriver = Optional.ofNullable(builder.manageMediaDriver).orElse(true); 31 | if (this.manageMediaDriver) { 32 | startMediaDriver(); 33 | } 34 | server = ReactiveSocketAeronServer.create(host, port, setupPayload -> { 35 | return new RxLmdbRequestHandler(db); 36 | }); 37 | } 38 | 39 | public void startMediaDriver() { 40 | ThreadingMode threadingMode = ThreadingMode.SHARED; 41 | 42 | boolean dedicated = Boolean.getBoolean("dedicated"); 43 | 44 | if (dedicated) { 45 | threadingMode = ThreadingMode.DEDICATED; 46 | } 47 | 48 | System.out.println("ThreadingMode => " + threadingMode); 49 | 50 | final MediaDriver.Context ctx = new MediaDriver.Context() 51 | .threadingMode(threadingMode) 52 | .dirsDeleteOnStart(true) 53 | .conductorIdleStrategy(new BackoffIdleStrategy(1, 1, 100, 1000)) 54 | .receiverIdleStrategy(new NoOpIdleStrategy()) 55 | .senderIdleStrategy(new NoOpIdleStrategy()); 56 | 57 | this.mediaDriver = MediaDriver.launch(ctx); 58 | } 59 | 60 | public void closeMediaDriver() { 61 | mediaDriver.close(); 62 | } 63 | 64 | public void close() throws Exception { 65 | server.close(); 66 | if (this.manageMediaDriver) { 67 | closeMediaDriver(); 68 | } 69 | } 70 | 71 | public static Builder builder() { 72 | return new Builder(); 73 | } 74 | 75 | public static class Builder { 76 | private Integer port; 77 | private String host; 78 | private RxLmdb lmdb; 79 | private RxDb db; 80 | private Boolean manageMediaDriver; 81 | 82 | public RxLmdbServer build() { 83 | return new RxLmdbServer(this); 84 | } 85 | 86 | public Builder host(String host) { 87 | this.host = host; 88 | return this; 89 | } 90 | 91 | public Builder port(int port) { 92 | this.port = port; 93 | return this; 94 | } 95 | 96 | public Builder lmdb(RxLmdb lmdb) { 97 | this.lmdb = lmdb; 98 | return this; 99 | } 100 | 101 | public Builder db(RxDb db) { 102 | this.db = db; 103 | return this; 104 | } 105 | 106 | public Builder manageMediaDriver(boolean shouldManage) { 107 | this.manageMediaDriver = shouldManage; 108 | return this; 109 | } 110 | 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /rxlmdb-grpc/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.deephacks.rxlmdb 5 | rxlmdb-project 6 | 0.0.5-SNAPSHOT 7 | 8 | 9 | 4.0.0 10 | rxlmdb-grpc 11 | rxlmdb-grpc 12 | jar 13 | rxlmdb-grpc 14 | 15 | 16 | 17 | org.deephacks.rxlmdb 18 | rxlmdb 19 | 0.0.5-SNAPSHOT 20 | 21 | 22 | org.deephacks.rxlmdb 23 | rxlmdb 24 | 0.0.5-SNAPSHOT 25 | test-jar 26 | test 27 | 28 | 29 | org.deephacks.lmdbjni 30 | lmdbjni-linux64 31 | 0.4.5 32 | 33 | 34 | io.grpc 35 | grpc-netty 36 | 0.13.2 37 | 38 | 39 | io.grpc 40 | grpc-stub 41 | 0.13.2 42 | 43 | 44 | io.grpc 45 | grpc-protobuf 46 | 0.13.2 47 | 48 | 49 | org.slf4j 50 | slf4j-simple 51 | 1.7.12 52 | test 53 | 54 | 55 | 56 | 57 | 58 | 59 | kr.motd.maven 60 | os-maven-plugin 61 | 1.4.1.Final 62 | 63 | 64 | 65 | 66 | org.xolstice.maven.plugins 67 | protobuf-maven-plugin 68 | 0.5.0 69 | 70 | 75 | com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} 76 | grpc-java 77 | io.grpc:protoc-gen-grpc-java:0.13.2:exe:${os.detected.classifier} 78 | 79 | 80 | 81 | 82 | compile 83 | compile-custom 84 | 85 | 86 | 87 | 88 | 89 | maven-compiler-plugin 90 | 3.1 91 | 92 | 1.8 93 | 1.8 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/DeleteTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | 8 | import java.util.Arrays; 9 | import java.util.LinkedList; 10 | 11 | import static com.google.common.truth.Truth.assertThat; 12 | import static org.deephacks.rxlmdb.Fixture.*; 13 | import static org.deephacks.rxlmdb.Fixture.__2; 14 | import static org.deephacks.rxlmdb.Fixture.__3; 15 | import static org.deephacks.rxlmdb.RxObservables.toStreamBlocking; 16 | import static org.junit.Assert.assertTrue; 17 | 18 | public class DeleteTest { 19 | RxDb db; 20 | RxLmdb lmdb; 21 | 22 | @Before 23 | public void before() { 24 | lmdb = RxLmdb.tmp(); 25 | db = lmdb.dbBuilder().build(); 26 | } 27 | 28 | @After 29 | public void after() { 30 | db.close(); 31 | db.lmdb.close(); 32 | } 33 | 34 | @Test 35 | public void testAbortDelete() { 36 | RxTx tx = lmdb.writeTx(); 37 | db.put(tx, Observable.from(_1_to_9)); 38 | tx.commit(); 39 | tx = lmdb.writeTx(); 40 | db.delete(tx, Observable.from(keys)); 41 | assertThat(toStreamBlocking(db.scan(tx, KeyRange.forward())).count()).isEqualTo(0L); 42 | tx.abort(); 43 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(9L); 44 | } 45 | 46 | @Test 47 | public void testCommitDelete() { 48 | RxTx tx = lmdb.writeTx(); 49 | db.put(tx, Observable.from(_1_to_9)); 50 | tx.commit(); 51 | tx = lmdb.writeTx(); 52 | db.delete(tx, Observable.from(keys)); 53 | assertThat(toStreamBlocking(db.scan(tx, KeyRange.forward())).count()).isEqualTo(0L); 54 | tx.commit(); 55 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(0L); 56 | } 57 | 58 | @Test 59 | public void testDeleteAll() { 60 | db.put(Observable.from(_1_to_9)); 61 | db.delete(); 62 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(0L); 63 | } 64 | 65 | @Test 66 | public void testDeleteAllAbort() { 67 | db.put(Observable.from(_1_to_9)); 68 | RxTx tx = lmdb.writeTx(); 69 | db.delete(tx); 70 | tx.abort(); 71 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(9L); 72 | } 73 | 74 | @Test 75 | public void testDeleteKeys() throws InterruptedException { 76 | db.put(Observable.from(_1_to_9)); 77 | db.delete(Observable.from(Arrays.asList(__2, __3))); 78 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(7L); 79 | db.delete(__4); 80 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(6L); 81 | } 82 | 83 | @Test 84 | public void testDeleteKeysAbort() { 85 | db.put(Observable.from(_1_to_9)); 86 | RxTx tx = lmdb.writeTx(); 87 | db.delete(tx, Observable.from(Arrays.asList(__2, __3))); 88 | tx.abort(); 89 | assertThat(toStreamBlocking(db.scan(KeyRange.forward())).count()).isEqualTo(9L); 90 | } 91 | 92 | @Test 93 | public void testDeleteRange() throws InterruptedException { 94 | RxTx tx = lmdb.writeTx(); 95 | db.put(tx, Observable.from(_1_to_9)); 96 | tx.commit(); 97 | tx = lmdb.writeTx(); 98 | Observable keys = db.scan(tx, KeyRange.atMost(new byte[]{5, 5})) 99 | .flatMap(Observable::from) 100 | .map(kv -> kv.key()); 101 | db.delete(tx, keys); 102 | tx.commit(); 103 | LinkedList expected = Fixture.range(__6, __9); 104 | toStreamBlocking(db.scan(KeyRange.forward())) 105 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 106 | assertTrue(expected.isEmpty()); 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/RxLmdbRequestHandler.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.Payload; 4 | import io.reactivesocket.RequestHandler; 5 | import org.reactivestreams.Publisher; 6 | import rx.Observable; 7 | import rx.RxReactiveStreams; 8 | import rx.subjects.PublishSubject; 9 | import rx.subjects.SerializedSubject; 10 | import uk.co.real_logic.agrona.concurrent.UnsafeBuffer; 11 | 12 | import java.util.List; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | class RxLmdbRequestHandler extends RequestHandler implements Loggable { 16 | RxDb db; 17 | SerializedSubject subject = PublishSubject.create().toSerialized(); 18 | Observable> buffer = subject.buffer(10, TimeUnit.MICROSECONDS, 512); 19 | 20 | RxLmdbRequestHandler(RxDb db) { 21 | this.db = db; 22 | this.db.batch(buffer); 23 | } 24 | 25 | @Override 26 | public Publisher handleRequestResponse(Payload payload) { 27 | UnsafeBuffer metadata = new UnsafeBuffer(payload.getMetadata()); 28 | OpType type = OpType.values()[metadata.getInt(0)]; 29 | if (type == OpType.PUT) { 30 | db.put(KeyValuePayload.getKeyValue(payload)); 31 | return s -> { 32 | s.onNext(Payloads.EMPTY_PAYLOAD); 33 | s.onComplete(); 34 | }; 35 | } else if (type == OpType.GET) { 36 | return s -> { 37 | byte[] key = KeyValuePayload.getByteArray(payload); 38 | byte[] val = db.get(key); 39 | if (val != null) { 40 | s.onNext(new KeyValuePayload(key, val, OpType.GET)); 41 | } else { 42 | s.onNext(Payloads.EMPTY_PAYLOAD); 43 | } 44 | s.onComplete(); 45 | }; 46 | } else if (type == OpType.DELETE) { 47 | return s -> { 48 | byte[] key = KeyValuePayload.getByteArray(payload); 49 | boolean success = db.delete(key); 50 | if (success) { 51 | s.onNext(new KeyValuePayload(key, OpType.DELETE)); 52 | } else { 53 | s.onNext(Payloads.EMPTY_PAYLOAD); 54 | } 55 | s.onComplete(); 56 | }; 57 | } else { 58 | logger().error("No OpType found for type {}", type); 59 | return s -> { 60 | s.onNext(Payloads.EMPTY_PAYLOAD); 61 | s.onComplete(); 62 | }; 63 | } 64 | } 65 | 66 | @Override 67 | public Publisher handleRequestStream(Payload payload) { 68 | UnsafeBuffer metadata = new UnsafeBuffer(payload.getMetadata()); 69 | OpType type = OpType.values()[metadata.getInt(0)]; 70 | if (type == OpType.SCAN) { 71 | return RxReactiveStreams.toPublisher(db.scan() 72 | .doOnError(throwable -> { 73 | throwable.printStackTrace(); 74 | }).flatMap(Observable::from) 75 | .map(kv -> { 76 | // need to clean up the payload 77 | return new KeyValuePayload(kv.key, kv.value, OpType.SCAN); 78 | })); 79 | } else { 80 | logger().error("No OpType found for type {}", type); 81 | return s -> { 82 | s.onNext(Payloads.EMPTY_PAYLOAD); 83 | s.onComplete(); 84 | }; 85 | } 86 | } 87 | 88 | @Override 89 | public Publisher handleFireAndForget(Payload payload) { 90 | return s -> { 91 | OpType type = OpType.values()[payload.getMetadata().getShort()]; 92 | if (type == OpType.PUT) { 93 | subject.onNext(KeyValuePayload.getKeyValue(payload)); 94 | } else { 95 | System.err.println(type + " not fireForget"); 96 | } 97 | s.onComplete(); 98 | }; 99 | } 100 | 101 | @Override 102 | public Publisher handleSubscription(Payload payload) { 103 | return RequestHandler.NO_REQUEST_SUBSCRIPTION_HANDLER.apply(payload); 104 | } 105 | 106 | @Override 107 | public Publisher handleChannel(Publisher payloads) { 108 | return RequestHandler.NO_REQUEST_CHANNEL_HANDLER.apply(payloads); 109 | } 110 | 111 | @Override 112 | public Publisher handleMetadataPush(Payload payload) { 113 | return RequestHandler.NO_METADATA_PUSH_HANDLER.apply(payload); 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/rx/internal/reactivestreams/RxJavaSynchronizedProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package rx.internal.reactivestreams; 17 | 18 | import org.reactivestreams.Subscription; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | public final class RxJavaSynchronizedProducer implements rx.Producer, rx.Subscription { 24 | private final Subscription subscription; 25 | private volatile boolean unsubscribed; 26 | /** Guarded by this. */ 27 | private boolean emitting; 28 | /** Guarded by this. */ 29 | private List requests; 30 | 31 | public RxJavaSynchronizedProducer(Subscription subscription) { 32 | if (subscription == null) { 33 | throw new NullPointerException("subscription"); 34 | } 35 | this.subscription = subscription; 36 | } 37 | @Override 38 | public boolean isUnsubscribed() { 39 | return unsubscribed; 40 | } 41 | @Override 42 | public void request(long n) { 43 | if (n > 0 && !unsubscribed) { 44 | synchronized (this) { 45 | if (unsubscribed) { 46 | return; 47 | } 48 | if (emitting) { 49 | if (requests == null) { 50 | requests = new ArrayList(4); 51 | } 52 | requests.add(n); 53 | return; 54 | } 55 | emitting = true; 56 | } 57 | boolean skipFinal = false; 58 | try { 59 | subscription.request(n); 60 | for (;;) { 61 | List list; 62 | synchronized (this) { 63 | list = requests; 64 | requests = null; 65 | if (list == null) { 66 | emitting = false; 67 | skipFinal = true; 68 | return; 69 | } 70 | } 71 | for (Long v : list) { 72 | if (v.longValue() == 0L) { 73 | unsubscribed = true; 74 | subscription.cancel(); 75 | skipFinal = true; 76 | return; 77 | } else { 78 | subscription.request(v); 79 | } 80 | } 81 | } 82 | } finally { 83 | if (!skipFinal) { 84 | synchronized (this) { 85 | emitting = false; 86 | } 87 | } 88 | } 89 | } 90 | } 91 | @Override 92 | public void unsubscribe() { 93 | if (!unsubscribed) { 94 | synchronized (this) { 95 | if (unsubscribed) { 96 | return; 97 | } 98 | if (emitting) { 99 | // replace all pending requests with this single cancel indicator 100 | requests = new ArrayList(4); 101 | requests.add(0L); 102 | return; 103 | } 104 | emitting = true; 105 | } 106 | unsubscribed = true; 107 | subscription.cancel(); 108 | // no need to leave emitting as this is a terminal state 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/org/deephacks/rxlmdb/RxLmdbClient.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.reactivesocket.ConnectionSetupPayload; 4 | import io.reactivesocket.ReactiveSocket; 5 | import io.reactivesocket.aeron.client.AeronClientDuplexConnection; 6 | import io.reactivesocket.aeron.client.AeronClientDuplexConnectionFactory; 7 | import org.reactivestreams.Publisher; 8 | import rx.Observable; 9 | import rx.RxReactiveStreams; 10 | 11 | import java.net.InetSocketAddress; 12 | import java.util.Arrays; 13 | import java.util.Optional; 14 | 15 | public class RxLmdbClient { 16 | static { 17 | System.setProperty("reactivesocket.aeron.clientConcurrency", "1"); 18 | } 19 | 20 | private ReactiveSocket reactiveSocket; 21 | private AeronClientDuplexConnection connection; 22 | private final InetSocketAddress address; 23 | private final AeronClientDuplexConnectionFactory cf; 24 | 25 | private RxLmdbClient(Builder builder) { 26 | String host = Optional.ofNullable(builder.host).orElse("localhost"); 27 | int port = Optional.ofNullable(builder.port).orElse(Consts.DEFAULT_PORT); 28 | this.address = new InetSocketAddress(host, port); 29 | this.cf = AeronClientDuplexConnectionFactory.getInstance(); 30 | cf.addSocketAddressToHandleResponses(address); 31 | } 32 | 33 | public RxLmdbClient connectAndWait() { 34 | Publisher udpConnection = cf.createAeronClientDuplexConnection(address); 35 | connection = RxReactiveStreams.toObservable(udpConnection) 36 | .toBlocking().single(); 37 | ConnectionSetupPayload setup = ConnectionSetupPayload.create("UTF-8", "UTF-8", ConnectionSetupPayload.NO_FLAGS); 38 | reactiveSocket = ReactiveSocket.fromClientConnection(connection, setup); 39 | reactiveSocket.startAndWait(); 40 | return this; 41 | } 42 | 43 | public void batch(KeyValue kv) { 44 | if (kv == null || kv.key == null || kv.key.length == 0) { 45 | return; 46 | } 47 | KeyValuePayload kvp = new KeyValuePayload(kv, OpType.PUT); 48 | RxReactiveStreams.toObservable(reactiveSocket.fireAndForget(kvp)) 49 | .map(payload -> { 50 | kvp.release(); 51 | return null; 52 | }).subscribe(); 53 | } 54 | 55 | public Observable put(KeyValue kv) { 56 | if (kv == null || kv.key == null || kv.key.length == 0) { 57 | return Observable.just(false); 58 | } 59 | KeyValuePayload kvp = new KeyValuePayload(kv, OpType.PUT); 60 | return RxReactiveStreams.toObservable(reactiveSocket.requestResponse(kvp)) 61 | .map(payload -> { 62 | kvp.release(); 63 | return Boolean.TRUE; 64 | }); 65 | } 66 | 67 | public Observable delete(byte[] key) { 68 | if (key == null || key.length == 0) { 69 | return Observable.just(false); 70 | } 71 | KeyValuePayload kvp = new KeyValuePayload(key, OpType.DELETE); 72 | return RxReactiveStreams.toObservable(reactiveSocket.requestResponse(kvp)) 73 | .map(payload -> { 74 | kvp.release(); 75 | byte[] response = KeyValuePayload.getByteArray(payload); 76 | return Arrays.equals(key, response); 77 | }); 78 | } 79 | 80 | public Observable get(byte[] key) { 81 | if (key == null || key.length == 0) { 82 | return Observable.just(null); 83 | } 84 | KeyValuePayload kvp = new KeyValuePayload(key, OpType.GET); 85 | return RxReactiveStreams 86 | .toObservable(reactiveSocket.requestResponse(kvp)) 87 | .map(payload -> { 88 | kvp.release(); 89 | return KeyValuePayload.getKeyValue(payload); 90 | }); 91 | } 92 | 93 | public Observable scan() { 94 | ScanPayload scan = new ScanPayload(); 95 | return RxReactiveStreams 96 | .toObservable(reactiveSocket.requestStream(scan)) 97 | .map(payload -> { 98 | KeyValue kv = KeyValuePayload.getKeyValue(payload); 99 | scan.release(); 100 | return kv; 101 | }); 102 | } 103 | 104 | public void close() throws Exception { 105 | reactiveSocket.close(); 106 | } 107 | 108 | public static Builder builder() { 109 | return new Builder(); 110 | } 111 | 112 | public static class Builder { 113 | private Integer port; 114 | private String host; 115 | 116 | public RxLmdbClient build() { 117 | return new RxLmdbClient(this); 118 | } 119 | 120 | public Builder host(String host) { 121 | this.host = host; 122 | return this; 123 | } 124 | 125 | public Builder port(int port) { 126 | this.port = port; 127 | return this; 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/DirectBufferComparator.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.DirectBuffer; 4 | 5 | import java.nio.ByteOrder; 6 | import java.util.Comparator; 7 | 8 | class DirectBufferComparator implements Comparator { 9 | public static final int LONG_BYTES = Long.SIZE / Byte.SIZE; 10 | static final boolean littleEndian = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); 11 | 12 | @Override 13 | public int compare(DirectBuffer o1, DirectBuffer o2) { 14 | return compareTo(o1, 0, o1.capacity(), o2, 0, o2.capacity()); 15 | } 16 | 17 | public static Comparator byteArrayComparator() { 18 | return (o1, o2) -> compareTo(o1, o2); 19 | } 20 | 21 | public static int compareTo(byte[] key, byte[] stop) { 22 | return compareTo(new DirectBuffer(key), new DirectBuffer(stop)); 23 | } 24 | 25 | public static boolean within(byte[] key, byte[] start, byte[] stop) { 26 | DirectBuffer startBuffer = new DirectBuffer(start); 27 | DirectBuffer stopBuffer = new DirectBuffer(stop); 28 | DirectBuffer keyBuffer = new DirectBuffer(key); 29 | if (compareTo(startBuffer, 0, stop.length, keyBuffer, 0, key.length) > 0) { 30 | return false; 31 | } 32 | if (compareTo(stopBuffer, 0, stop.length, keyBuffer, 0, key.length) < 0) { 33 | return false; 34 | } 35 | return true; 36 | } 37 | 38 | public static int compareTo(DirectBuffer key, DirectBuffer stop) { 39 | if (stop.capacity() > key.capacity()) { 40 | return -1; 41 | } 42 | return compareTo(key, 0, stop.capacity(), stop, 0, stop.capacity()); 43 | } 44 | 45 | /** 46 | * Lexicographically compare two arrays. 47 | * 48 | * @param buffer1 left operand 49 | * @param buffer2 right operand 50 | * @param offset1 Where to beginTransaction comparing in the left buffer 51 | * @param offset2 Where to beginTransaction comparing in the right buffer 52 | * @param length1 How much to compare from the left buffer 53 | * @param length2 How much to compare from the right buffer 54 | * @return 0 if equal, less 0 if left is less than right, etc. 55 | */ 56 | public static int compareTo(DirectBuffer buffer1, int offset1, int length1, 57 | DirectBuffer buffer2, int offset2, int length2) { 58 | // Short circuit equal case 59 | if (buffer1 == buffer2 && 60 | offset1 == offset2 && 61 | length1 == length2) { 62 | return 0; 63 | } 64 | int minLength = Math.min(length1, length2); 65 | int minWords = minLength / LONG_BYTES; 66 | 67 | /* 68 | * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a 69 | * time is no slower than comparing 4 bytes at a time even on 32-bit. 70 | * On the other hand, it is substantially faster on 64-bit. 71 | */ 72 | for (int i = 0; i < minWords * LONG_BYTES; i += LONG_BYTES) { 73 | long lw = buffer1.getLong(i, ByteOrder.BIG_ENDIAN); 74 | long rw = buffer2.getLong(i, ByteOrder.BIG_ENDIAN); 75 | long diff = lw ^ rw; 76 | 77 | if (diff != 0) { 78 | if (!littleEndian) { 79 | return lessThanUnsigned(lw, rw) ? -1 : 1; 80 | } 81 | 82 | // Use binary search 83 | int n = 0; 84 | int y; 85 | int x = (int) diff; 86 | if (x == 0) { 87 | x = (int) (diff >>> 32); 88 | n = 32; 89 | } 90 | 91 | y = x << 16; 92 | if (y == 0) { 93 | n += 16; 94 | } else { 95 | x = y; 96 | } 97 | 98 | y = x << 8; 99 | if (y == 0) { 100 | n += 8; 101 | } 102 | return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL)); 103 | } 104 | } 105 | 106 | // The epilogue to cover the last (minLength % 8) elements. 107 | for (int i = minWords * LONG_BYTES; i < minLength; i++) { 108 | int result = compare( 109 | buffer1.getByte(offset1 + i), 110 | buffer2.getByte(offset2 + i)); 111 | if (result != 0) { 112 | return result; 113 | } 114 | } 115 | return length1 - length2; 116 | } 117 | 118 | public static int compare(byte a, byte b) { 119 | return toInt(a) - toInt(b); 120 | } 121 | 122 | /** 123 | * Returns true if x1 is less than x2, when both values are treated as 124 | * unsigned. 125 | */ 126 | static boolean lessThanUnsigned(long x1, long x2) { 127 | return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE); 128 | } 129 | 130 | public static int toInt(byte value) { 131 | return value & 0xFF; 132 | } 133 | } -------------------------------------------------------------------------------- /rxlmdb-aeron/src/main/java/rx/internal/reactivestreams/PublisherAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package rx.internal.reactivestreams; 17 | 18 | import org.reactivestreams.Publisher; 19 | import org.reactivestreams.Subscriber; 20 | import org.reactivestreams.Subscription; 21 | import rx.Observable; 22 | import rx.internal.operators.BackpressureUtils; 23 | 24 | import java.util.concurrent.ConcurrentHashMap; 25 | import java.util.concurrent.ConcurrentMap; 26 | import java.util.concurrent.atomic.AtomicBoolean; 27 | import java.util.concurrent.atomic.AtomicLong; 28 | 29 | public class PublisherAdapter implements Publisher { 30 | 31 | private final Observable observable; 32 | 33 | private final ConcurrentMap, Object> subscribers = new ConcurrentHashMap, Object>(); 34 | 35 | public PublisherAdapter(final Observable observable) { 36 | this.observable = observable.serialize(); 37 | } 38 | 39 | @Override 40 | public void subscribe(final Subscriber s) { 41 | if (subscribers.putIfAbsent(s, s) == null) { 42 | observable.subscribe(new rx.Subscriber() { 43 | private final AtomicBoolean done = new AtomicBoolean(); 44 | private final AtomicLong childRequested = new AtomicLong(); 45 | private void doRequest(long n) { 46 | if (!done.get()) { 47 | BackpressureUtils.getAndAddRequest(childRequested, n); 48 | request(n); 49 | } 50 | } 51 | 52 | @Override 53 | public void onStart() { 54 | final AtomicBoolean requested = new AtomicBoolean(); 55 | s.onSubscribe(new Subscription() { 56 | @Override 57 | public void request(long n) { 58 | if (n < 1) { 59 | unsubscribe(); 60 | onError(new IllegalArgumentException("3.9 While the Subscription is not cancelled, Subscription.request(long n) MUST throw a java.lang.IllegalArgumentException if the argument is <= 0.")); 61 | return; 62 | } 63 | 64 | requested.set(true); 65 | doRequest(n); 66 | } 67 | 68 | @Override 69 | public void cancel() { 70 | unsubscribe(); 71 | fireDone(); 72 | } 73 | }); 74 | 75 | if (!requested.get()) { 76 | request(0); 77 | } 78 | } 79 | 80 | private boolean fireDone() { 81 | boolean first = done.compareAndSet(false, true); 82 | if (first) { 83 | subscribers.remove(s); 84 | } 85 | return first; 86 | } 87 | 88 | @Override 89 | public void onCompleted() { 90 | if (fireDone()) { 91 | s.onComplete(); 92 | } 93 | } 94 | 95 | @Override 96 | public void onError(Throwable e) { 97 | if (fireDone()) { 98 | s.onError(e); 99 | } 100 | } 101 | 102 | @Override 103 | public void onNext(T t) { 104 | if (!done.get()) { 105 | if (childRequested.get() > 0) { 106 | s.onNext(t); 107 | childRequested.decrementAndGet(); 108 | } else { 109 | try { 110 | onError(new IllegalStateException("1.1 source doesn't respect backpressure")); 111 | } finally { 112 | unsubscribe(); 113 | } 114 | } 115 | } 116 | } 117 | }); 118 | } else { 119 | s.onError(new IllegalArgumentException("1.10 Subscriber cannot subscribe more than once")); 120 | } 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/main/java/org/deephacks/rxlmdb/RxDbServiceGrpc.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import com.google.protobuf.ByteString; 4 | import io.grpc.Status; 5 | import io.grpc.StatusRuntimeException; 6 | import io.grpc.stub.StreamObserver; 7 | import org.deephacks.rxlmdb.Rxdb.*; 8 | import org.fusesource.lmdbjni.LMDBException; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import rx.Observable; 12 | import rx.Observer; 13 | import rx.subjects.PublishSubject; 14 | import rx.subjects.SerializedSubject; 15 | 16 | import java.util.List; 17 | import java.util.concurrent.TimeUnit; 18 | 19 | final class RxDbServiceGrpc implements DatabaseServiceGrpc.DatabaseService { 20 | private static final Logger logger = LoggerFactory.getLogger(RxDbServiceGrpc.class); 21 | private static final KeyRange.KeyRangeType[] TYPES = KeyRange.KeyRangeType.values(); 22 | private RxLmdb lmdb; 23 | private RxDb db; 24 | 25 | RxDbServiceGrpc(RxDb db) { 26 | this.lmdb = db.lmdb; 27 | this.db = db; 28 | } 29 | 30 | @Override 31 | public void put(PutMsg msg, StreamObserver response) { 32 | try { 33 | db.put(new KeyValue(msg.getKey().toByteArray(), msg.getVal().toByteArray())); 34 | response.onCompleted(); 35 | } catch (LMDBException e) { 36 | if (logger.isErrorEnabled()) { 37 | logger.error("LMDB error " + e.getErrorCode() + " '" + e.getMessage() + "'"); 38 | } 39 | throw new StatusRuntimeException(Status.INTERNAL); 40 | } 41 | } 42 | 43 | @Override 44 | public StreamObserver batch(StreamObserver response) { 45 | SerializedSubject subject = PublishSubject.create().toSerialized(); 46 | Observable> buffer = subject.buffer(10, TimeUnit.NANOSECONDS, 512); 47 | db.batch(buffer); 48 | return new StreamObserver() { 49 | @Override 50 | public void onNext(PutMsg msg) { 51 | subject.onNext(new KeyValue(msg.getKey().toByteArray(), msg.getVal().toByteArray())); 52 | } 53 | 54 | @Override 55 | public void onError(Throwable t) { 56 | subject.onError(t); 57 | } 58 | 59 | @Override 60 | public void onCompleted() { 61 | subject.onCompleted(); 62 | } 63 | }; 64 | } 65 | 66 | @Override 67 | public void get(Rxdb.GetMsg msg, StreamObserver response) { 68 | try { 69 | byte[] bytes = db.get(msg.getKey().toByteArray()); 70 | ValueMsg.Builder builder = ValueMsg.newBuilder(); 71 | if (bytes != null) { 72 | builder.setVal(ByteString.copyFrom(bytes)); 73 | } 74 | response.onNext(builder.build()); 75 | response.onCompleted(); 76 | } catch (LMDBException e) { 77 | if (logger.isErrorEnabled()) { 78 | logger.error("LMDB error " + e.getErrorCode() + " '" + e.getMessage() + "'"); 79 | } 80 | throw new StatusRuntimeException(Status.INTERNAL); 81 | } 82 | } 83 | 84 | @Override 85 | public void delete(DeleteMsg msg, StreamObserver response) { 86 | try { 87 | boolean delete = db.delete(msg.getKey().toByteArray()); 88 | response.onNext(BooleanMsg.newBuilder().setValue(delete).build()); 89 | response.onCompleted(); 90 | } catch (LMDBException e) { 91 | if (logger.isErrorEnabled()) { 92 | logger.error("LMDB error " + e.getErrorCode() + " '" + e.getMessage() + "'"); 93 | } 94 | throw new StatusRuntimeException(Status.INTERNAL); 95 | } 96 | } 97 | 98 | @Override 99 | public void scan(KeyRangeMsg msg, StreamObserver response) { 100 | KeyRange.KeyRangeType type = TYPES[msg.getType().getNumber()]; 101 | KeyRange keyRange = new KeyRange(msg.getStart().toByteArray(), msg.getStop().toByteArray(), type); 102 | try { 103 | db.scan(keyRange).subscribe(new Observer>() { 104 | @Override 105 | public void onCompleted() { 106 | response.onCompleted(); 107 | } 108 | 109 | @Override 110 | public void onError(Throwable throwable) { 111 | if (throwable instanceof LMDBException) { 112 | if (logger.isErrorEnabled()) { 113 | LMDBException e = (LMDBException) throwable; 114 | logger.error("LMDB error " + e.getErrorCode() + " '" + e.getMessage() + "'"); 115 | } 116 | response.onError(new StatusRuntimeException(Status.INTERNAL)); 117 | } else { 118 | response.onError(throwable); 119 | } 120 | } 121 | 122 | @Override 123 | public void onNext(List kvs) { 124 | for (KeyValue kv : kvs) { 125 | ByteString k = ByteString.copyFrom(kv.key()); 126 | ByteString v = ByteString.copyFrom(kv.value()); 127 | KeyValueMsg msg = KeyValueMsg.newBuilder().setKey(k).setVal(v).build(); 128 | response.onNext(msg); 129 | } 130 | } 131 | }); 132 | } catch (LMDBException e) { 133 | if (logger.isErrorEnabled()) { 134 | logger.error("LMDB error " + e.getErrorCode() + " '" + e.getMessage() + "'"); 135 | } 136 | throw new StatusRuntimeException(Status.INTERNAL); 137 | } 138 | } 139 | } -------------------------------------------------------------------------------- /rxlmdb-grpc/src/test/java/org/deephacks/rxlmdb/CrudTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import io.grpc.Status; 4 | import io.grpc.StatusRuntimeException; 5 | import org.junit.*; 6 | import rx.schedulers.Schedulers; 7 | import rx.subjects.PublishSubject; 8 | import rx.subjects.SerializedSubject; 9 | 10 | import java.io.IOException; 11 | import java.nio.file.Path; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.Observable; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | import static org.hamcrest.core.Is.is; 18 | import static org.junit.Assert.*; 19 | 20 | public class CrudTest { 21 | 22 | RxDbGrpcServer server; 23 | RxDbGrpcClient client; 24 | RxLmdb lmdb; 25 | RxDb db; 26 | 27 | @Before 28 | public void before() throws IOException { 29 | lmdb = RxLmdb.tmp(); 30 | db = lmdb.dbBuilder().build(); 31 | server = RxDbGrpcServer.builder().lmdb(lmdb).db(db).build(); 32 | client = RxDbGrpcClient.builder().build(); 33 | } 34 | 35 | @After 36 | public void after() throws Exception { 37 | client.close(); 38 | server.close(); 39 | } 40 | 41 | @Test 42 | public void testPutGetDelete() { 43 | KeyValue kv1 = Fixture.values[0]; 44 | assertTrue(client.put(kv1).toBlocking().first()); 45 | KeyValue kv2 = null; 46 | for (int i = 0; i < 100; i++) { 47 | kv2 = client.get(kv1.key()).toBlocking().first(); 48 | } 49 | assertArrayEquals(kv1.key(), kv2.key()); 50 | assertArrayEquals(kv1.value(), kv2.value()); 51 | 52 | assertTrue(client.delete(kv1.key()).toBlocking().first()); 53 | assertNull(client.get(kv1.key()).toBlocking().firstOrDefault(null)); 54 | } 55 | 56 | @Test 57 | public void testPutServerFailure() { 58 | db.close(); 59 | try { 60 | KeyValue kv1 = Fixture.values[0]; 61 | client.put(kv1).toBlocking().first(); 62 | fail("should fail"); 63 | } catch (StatusRuntimeException e) { 64 | assertThat(e.getStatus(), is(Status.INTERNAL)); 65 | } 66 | } 67 | 68 | @Test 69 | public void testGetServerFailure() { 70 | db.close(); 71 | try { 72 | client.get(new byte[1]).toBlocking().first(); 73 | fail("should fail"); 74 | } catch (StatusRuntimeException e) { 75 | assertThat(e.getStatus(), is(Status.INTERNAL)); 76 | } 77 | } 78 | 79 | @Test 80 | public void testDeleteServerFailure() { 81 | db.close(); 82 | try { 83 | client.delete(new byte[1]).toBlocking().first(); 84 | fail("should fail"); 85 | } catch (StatusRuntimeException e) { 86 | assertThat(e.getStatus(), is(Status.INTERNAL)); 87 | } 88 | } 89 | 90 | @Test 91 | public void deleteNonExisting() { 92 | assertFalse(client.delete(new byte[] {9,9,9}).toBlocking().first()); 93 | } 94 | 95 | @Test 96 | public void getNonExisting() { 97 | KeyValue keyValue = client.get(new byte[]{9, 9, 9}).toBlocking().firstOrDefault(null); 98 | assertNull(keyValue); 99 | } 100 | 101 | @Test 102 | public void getEmptyKey() { 103 | assertNull(client.get(new byte[0]).toBlocking().firstOrDefault(null)); 104 | } 105 | 106 | @Test 107 | public void testBatch() throws InterruptedException { 108 | List kvs = new ArrayList<>(); 109 | SerializedSubject subject = PublishSubject.create().toSerialized(); 110 | client.batch(subject.buffer(10, TimeUnit.NANOSECONDS, 512)); 111 | for (int i = 0; i < 1000; i++) { 112 | KeyValue kv = Fixture.kv(i, i); 113 | kvs.add(kv); 114 | subject.onNext(kv); 115 | } 116 | subject.onCompleted(); 117 | Thread.sleep(1000); 118 | 119 | List list = client.scan() 120 | .toList().toBlocking().first(); 121 | assertThat(list.size(), is(1000)); 122 | for (int i = 0; i < 1000; i++) { 123 | KeyValue kv = kvs.get(i); 124 | assertThat(new String(kv.key()), is(new String(list.get(i).key()))); 125 | } 126 | } 127 | 128 | @Test 129 | public void testBatchServerFailure() throws InterruptedException { 130 | db.close(); 131 | SerializedSubject subject = PublishSubject.create().toSerialized(); 132 | client.batch(subject.buffer(10, TimeUnit.NANOSECONDS, 512)); 133 | subject.onNext(Fixture.kv(1, 1)); 134 | subject.onCompleted(); 135 | } 136 | 137 | @Test 138 | public void testScan() throws InterruptedException { 139 | List kvs = new ArrayList<>(); 140 | for (int i = 0; i < 100; i++) { 141 | KeyValue kv = Fixture.kv(i, i); 142 | kvs.add(kv); 143 | assertTrue(client.put(kv).toBlocking().first()); 144 | } 145 | List list = client.scan() 146 | .toList().toBlocking().first(); 147 | assertThat(list.size(), is(100)); 148 | for (int i = 0; i < 100; i++) { 149 | KeyValue kv = kvs.get(i); 150 | assertThat(new String(kv.key()), is(new String(list.get(i).key()))); 151 | } 152 | } 153 | 154 | @Test 155 | public void testScanServerFailure() throws InterruptedException { 156 | db.close(); 157 | try { 158 | client.scan().toList().toBlocking().first(); 159 | fail("should fail"); 160 | } catch (StatusRuntimeException e) { 161 | assertThat(e.getStatus(), is(Status.INTERNAL)); 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/RxLmdb.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import org.fusesource.lmdbjni.*; 17 | import rx.Scheduler; 18 | import rx.schedulers.Schedulers; 19 | 20 | import java.nio.file.Path; 21 | import java.nio.file.Paths; 22 | import java.util.Optional; 23 | 24 | public class RxLmdb { 25 | final Env env; 26 | final Path path; 27 | final Scheduler scheduler; 28 | final int flags; 29 | 30 | private RxLmdb(Builder builder) { 31 | this.env = new Env(); 32 | Optional.ofNullable(builder.size) 33 | .ifPresent(size -> this.env.setMapSize(size)); 34 | Optional.ofNullable(builder.maxDbs) 35 | .ifPresent(size -> this.env.setMaxDbs(builder.maxDbs)); 36 | Optional.ofNullable(builder.maxReaders) 37 | .ifPresent(size -> this.env.setMaxReaders(builder.maxReaders)); 38 | this.flags = builder.flags; 39 | this.path = IoUtil.createPathOrTemp(builder.path); 40 | 41 | // never tie transactions to threads since it breaks parallel range scans 42 | this.env.open(path.toString(), Constants.NOTLS | builder.flags); 43 | this.scheduler = Optional.ofNullable(builder.scheduler) 44 | .orElse(Schedulers.io()); 45 | } 46 | 47 | public static Builder builder() { 48 | return new Builder(); 49 | } 50 | 51 | /** 52 | * Creates an environment of 64MB in a temporary location. 53 | */ 54 | public static RxLmdb tmp() { 55 | return new Builder() 56 | .size(64, ByteUnit.MEBIBYTES) 57 | .maxDbs(12) 58 | .build(); 59 | } 60 | 61 | public Path getPath() { 62 | return path; 63 | } 64 | 65 | public long getSize() { 66 | return env.info().getMapSize(); 67 | } 68 | 69 | public void copy(String path) { 70 | env.copy(path); 71 | } 72 | 73 | public void copyCompact(String path) { 74 | env.copyCompact(path); 75 | } 76 | 77 | public void sync(boolean force) { 78 | env.sync(force); 79 | } 80 | 81 | public void close() { 82 | env.close(); 83 | } 84 | 85 | public RxTx writeTx() { 86 | return new RxTx(env.createWriteTransaction(), true); 87 | } 88 | 89 | public RxTx readTx() { 90 | return new RxTx(env.createReadTransaction(), true); 91 | } 92 | 93 | RxTx internalReadTx() { 94 | return new RxTx(env.createReadTransaction(), false); 95 | } 96 | 97 | RxTx internalWriteTx() { 98 | return new RxTx(env.createWriteTransaction(), false); 99 | } 100 | 101 | public RxDb.Builder dbBuilder() { 102 | return RxDb.builder().lmdb(this); 103 | } 104 | 105 | public static class Builder { 106 | private Path path; 107 | private long size; 108 | private Scheduler scheduler; 109 | private int flags; 110 | private Long maxDbs; 111 | private Long maxReaders; 112 | 113 | public Builder path(String path) { 114 | this.path = Paths.get(path); 115 | return this; 116 | } 117 | 118 | public Builder path(Path path) { 119 | this.path = path; 120 | return this; 121 | } 122 | 123 | public Builder size(long size, ByteUnit unit) { 124 | this.size = unit.toBytes(size); 125 | return this; 126 | } 127 | 128 | public Builder scheduler(Scheduler scheduler) { 129 | this.scheduler = scheduler; 130 | return this; 131 | } 132 | 133 | public Builder fixedmap() { 134 | flags |= Constants.FIXEDMAP; 135 | return this; 136 | } 137 | 138 | public Builder noSubdir() { 139 | flags |= Constants.NOSUBDIR; 140 | return this; 141 | } 142 | 143 | public Builder readOnly() { 144 | flags |= Constants.RDONLY; 145 | return this; 146 | } 147 | 148 | public Builder writeMap() { 149 | flags |= Constants.WRITEMAP; 150 | return this; 151 | } 152 | 153 | public Builder noMetaSync() { 154 | flags |= Constants.NOMETASYNC; 155 | return this; 156 | } 157 | 158 | public Builder noSync() { 159 | flags |= Constants.NOSYNC; 160 | return this; 161 | } 162 | 163 | public Builder mapAsync() { 164 | flags |= Constants.MAPASYNC; 165 | return this; 166 | } 167 | 168 | public Builder noLock() { 169 | flags |= Constants.NOLOCK; 170 | return this; 171 | } 172 | 173 | public Builder noReadahead() { 174 | flags |= Constants.NORDAHEAD; 175 | return this; 176 | } 177 | 178 | public Builder noMemInit() { 179 | flags |= Constants.NOMEMINIT; 180 | return this; 181 | } 182 | 183 | public Builder maxDbs(long size) { 184 | maxDbs = size; 185 | return this; 186 | } 187 | 188 | public Builder maxReaders(long size) { 189 | maxReaders = size; 190 | return this; 191 | } 192 | 193 | public RxLmdb build() { 194 | return new RxLmdb(this); 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /jmh/src/main/java/org/deephacks/rxlmdb/RangedRowsSetup.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import generated.proto.Address; 4 | import generated.proto.User; 5 | import okio.ByteString; 6 | import org.fusesource.lmdbjni.ByteUnit; 7 | import org.fusesource.lmdbjni.DirectBuffer; 8 | import rx.Observable; 9 | import rx.Subscriber; 10 | 11 | import java.io.IOException; 12 | import java.nio.ByteOrder; 13 | import java.nio.file.Files; 14 | import java.nio.file.Path; 15 | import java.nio.file.Paths; 16 | import java.util.function.Function; 17 | 18 | public class RangedRowsSetup { 19 | public RxDb db; 20 | public RxLmdb lmdb; 21 | public KeyRange[] keyRanges = new KeyRange[16]; 22 | 23 | public static User[] PROTO = new User[10]; 24 | 25 | static { 26 | for (int i = 0; i < PROTO.length; i++) { 27 | Address address = new Address.Builder() 28 | .areaCode(i) 29 | .country(i) 30 | .street(ByteString.of(("street " + i).getBytes())) 31 | .telephone(Long.valueOf(i)) 32 | .zipcode(i) 33 | .build(); 34 | PROTO[i] = new User.Builder() 35 | .ssn(ByteString.of(new byte[]{1, 2, 3, 4, 5, (byte) i})) 36 | .firstname(ByteString.of(("name" + i).getBytes())) 37 | .lastname(ByteString.of(("lastname" + i).getBytes())) 38 | .email(ByteString.of(("email" + i + "@email.com").getBytes())) 39 | .mobile(Long.valueOf(i)) 40 | .address(address) 41 | .build(); 42 | } 43 | } 44 | 45 | public static UserVal[] VALS = new UserVal[10]; 46 | 47 | static { 48 | for (int i = 0; i < VALS.length; i++) { 49 | AddressVal address = new AddressValBuilder() 50 | .withAreaCode(i) 51 | .withCountry(Country.SE) 52 | .withStreetname(("street " + i).getBytes()) 53 | .withTelephone(i) 54 | .withZipcode(i) 55 | .build(); 56 | VALS[i] = new UserValBuilder() 57 | .withSsn(new byte[]{1, 2, 3, 4, 5, (byte) i}) 58 | .withFirstname(("name" + i).getBytes()) 59 | .withLastname(("lastname" + i).getBytes()) 60 | .withEmail(("email" + i + "@email.com").getBytes()) 61 | .withMobile(Long.valueOf(i)) 62 | .withAddress(address) 63 | .build(); 64 | } 65 | } 66 | 67 | public static uk.co.real_logic.sbe.codec.java.DirectBuffer[] SBE = 68 | new uk.co.real_logic.sbe.codec.java.DirectBuffer[10]; 69 | 70 | static { 71 | for (int i = 0; i < SBE.length; i++) { 72 | uk.co.real_logic.sbe.codec.java.DirectBuffer buffer = new uk.co.real_logic.sbe.codec.java.DirectBuffer(new byte[64]); 73 | SBE[i] = buffer; 74 | generated.sbe.User user = new generated.sbe.User(); 75 | user.wrapForEncode(buffer, 0); 76 | byte[] firstname = ("name" + i).getBytes(); 77 | byte[] lastname = ("lastname" + i).getBytes(); 78 | byte[] email = ("email" + i + "@email.com").getBytes(); 79 | user.mobile(i); 80 | user.putFirstname(firstname, 0, firstname.length); 81 | user.putLastname(lastname, 0, lastname.length); 82 | user.putEmail(email, 0, email.length); 83 | } 84 | } 85 | 86 | public RangedRowsSetup(Class testcase) { 87 | Path path = Paths.get("/tmp/rxlmdb-jmh-" + testcase.getSimpleName()); 88 | try { 89 | Files.createDirectories(path); 90 | lmdb = RxLmdb.builder().path(path).size(1, ByteUnit.GIBIBYTES).build(); 91 | db = RxDb.builder().lmdb(lmdb).build(); 92 | } catch (IOException e) { 93 | throw new RuntimeException(e); 94 | } 95 | for (int i = 0; i < keyRanges.length; i++) { 96 | keyRanges[i] = KeyRange.range(new byte[]{(byte) i}, new byte[]{(byte) i}); 97 | } 98 | } 99 | 100 | public void writeSmallKeyValue() { 101 | writeRanges(i -> { 102 | DirectBuffer val = new DirectBuffer(new byte[5]); 103 | val.putInt(0, i); 104 | return val.byteArray(); 105 | }); 106 | } 107 | 108 | public void writeBigKeyValue() { 109 | final byte[] bytes = new byte[1024]; 110 | writeRanges(i -> bytes); 111 | } 112 | 113 | public void writeProto() { 114 | writeRanges(i -> PROTO[i % PROTO.length].toByteArray()); 115 | } 116 | 117 | public void writeValsRanges() { 118 | writeRanges(i -> VALS[i % VALS.length].toByteArray()); 119 | } 120 | 121 | public void writeSbeRanges() { 122 | writeRanges(i -> SBE[i % SBE.length].array()); 123 | } 124 | 125 | private void writeRanges(Function value) { 126 | if (db.scan().count().toBlocking().first() != 0) { 127 | return; 128 | } 129 | Observable observable = Observable.create(new Observable.OnSubscribe() { 130 | @Override 131 | public void call(Subscriber subscriber) { 132 | DirectBuffer key = new DirectBuffer(new byte[5]); 133 | for (int j = 0; j < keyRanges.length; j++) { 134 | for (int i = 0; i < 100_000; i++) { 135 | byte[] bytes = value.apply(i); 136 | DirectBuffer val = new DirectBuffer(new byte[bytes.length]); 137 | key.putByte(0, (byte) j); 138 | key.putInt(1, (byte) i, ByteOrder.BIG_ENDIAN); 139 | val.putBytes(0, bytes); 140 | subscriber.onNext(new KeyValue(key.byteArray(), val.byteArray())); 141 | } 142 | } 143 | subscriber.onCompleted(); 144 | } 145 | }).doOnError(throwable -> throwable.printStackTrace()); 146 | db.put(observable); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/test/java/org/deephacks/rxlmdb/ScanTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.*; 4 | import rx.Observable; 5 | 6 | import java.util.LinkedList; 7 | import java.util.NoSuchElementException; 8 | 9 | import static com.google.common.truth.Truth.assertThat; 10 | import static org.deephacks.rxlmdb.Fixture.*; 11 | 12 | public class ScanTest { 13 | static RxDbGrpcServer server; 14 | static RxDbGrpcClient client; 15 | 16 | @BeforeClass 17 | public static void beforeClass() throws Exception { 18 | server = RxDbGrpcServer.builder().build(); 19 | client = RxDbGrpcClient.builder().build(); 20 | client.batch(Observable.from(_1_to_9).buffer(10)); 21 | Thread.sleep(100); 22 | } 23 | 24 | @AfterClass 25 | public static void afterClass() throws Exception { 26 | client.close(); 27 | server.close(); 28 | } 29 | 30 | @Test 31 | public void testScanRangeForward() { 32 | LinkedList expected = Fixture.range(__2, __3); 33 | client.scan(KeyRange.range(_2, _3)).toBlocking() 34 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 35 | assertThat(expected).isEqualTo(new LinkedList<>()); 36 | } 37 | 38 | @Test 39 | public void testCount() { 40 | Integer count = client.scan().count().toBlocking().first(); 41 | assertThat(count).isEqualTo(9); 42 | } 43 | 44 | @Test 45 | public void testScanSingleRange() { 46 | LinkedList expected = Fixture.range(__1, __1); 47 | client.scan(KeyRange.range(_1, _1)).toBlocking() 48 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 49 | assertThat(expected).isEqualTo(new LinkedList<>()); 50 | } 51 | 52 | @Test 53 | public void testScanOnly() { 54 | byte[] startStop = new byte[] { (byte) 2 }; 55 | LinkedList expected = Fixture.range(__2, __2); 56 | client.scan(KeyRange.range(startStop, startStop)).toBlocking() 57 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 58 | assertThat(expected).isEqualTo(new LinkedList<>()); 59 | } 60 | 61 | @Test 62 | public void testScanRangeBackward() { 63 | LinkedList expected = Fixture.range(__3, __2); 64 | client.scan(KeyRange.range(_3, _2)).toBlocking() 65 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 66 | assertThat(expected).isEqualTo(new LinkedList<>()); 67 | } 68 | 69 | @Test 70 | public void testScanAtLeast() { 71 | LinkedList expected = Fixture.range(__3, __9); 72 | client.scan(KeyRange.atLeast(_3)).toBlocking() 73 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 74 | assertThat(expected).isEqualTo(new LinkedList<>()); 75 | } 76 | 77 | @Test 78 | public void testScanAtLeastBackward() { 79 | LinkedList expected = Fixture.range(__3, __1); 80 | client.scan(KeyRange.atLeastBackward(__3)).toBlocking() 81 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 82 | assertThat(expected).isEqualTo(new LinkedList<>()); 83 | } 84 | 85 | @Test 86 | public void testScanAtMost() { 87 | LinkedList expected = Fixture.range(__1, __4); 88 | client.scan(KeyRange.atMost(_4)).toBlocking() 89 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 90 | assertThat(expected).isEqualTo(new LinkedList<>()); 91 | } 92 | 93 | @Test 94 | public void testScanAtMostBackward() { 95 | LinkedList expected = Fixture.range(__9, __4); 96 | client.scan(KeyRange.atMostBackward(_4)).toBlocking() 97 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 98 | assertThat(expected).isEqualTo(new LinkedList<>()); 99 | } 100 | 101 | @Test 102 | public void testScanForward() { 103 | LinkedList expected = Fixture.range(__1, __9); 104 | client.scan(KeyRange.forward()).toBlocking() 105 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 106 | assertThat(expected).isEqualTo(new LinkedList<>()); 107 | } 108 | 109 | @Test 110 | public void testScanForwardUnsubscribe() throws InterruptedException { 111 | LinkedList expected = Fixture.range(__1, __6); 112 | client.scan(KeyRange.forward()).takeWhile(kv -> { 113 | byte[] key = expected.pollFirst().key(); 114 | assertThat(key).isEqualTo(kv.key()); 115 | return key[0] < 6; 116 | }).toBlocking().last(); 117 | assertThat(expected).isEqualTo(new LinkedList<>()); 118 | } 119 | 120 | 121 | @Test 122 | public void testScanBackward() { 123 | LinkedList expected = Fixture.range(__9, __1); 124 | client.scan(KeyRange.backward()).toBlocking() 125 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 126 | assertThat(expected).isEqualTo(new LinkedList<>()); 127 | } 128 | 129 | @Test 130 | public void testScanBackwardUnsubscribe() throws InterruptedException { 131 | LinkedList expected = Fixture.range(__9, __6); 132 | client.scan(KeyRange.backward()).takeWhile(kv -> { 133 | byte[] key = expected.pollFirst().key(); 134 | assertThat(key).isEqualTo(kv.key()); 135 | return key[0] > 6; 136 | }).toBlocking().last(); 137 | assertThat(expected).isEqualTo(new LinkedList<>()); 138 | } 139 | 140 | @Test(expected = NoSuchElementException.class) 141 | public void testScanEmptyRange() { 142 | client.scan(KeyRange.range(new byte[]{12}, new byte[]{123})).toBlocking().first(); 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/PutTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | import rx.Observer; 8 | import rx.functions.Action1; 9 | import rx.schedulers.Schedulers; 10 | 11 | import java.util.LinkedList; 12 | import java.util.NoSuchElementException; 13 | import java.util.concurrent.atomic.AtomicReference; 14 | 15 | import static com.google.common.truth.Truth.assertThat; 16 | import static org.deephacks.rxlmdb.Fixture.*; 17 | import static org.deephacks.rxlmdb.Fixture.__3; 18 | import static org.deephacks.rxlmdb.Fixture.values; 19 | import static org.deephacks.rxlmdb.RxObservables.toStreamBlocking; 20 | import static org.junit.Assert.assertArrayEquals; 21 | import static org.junit.Assert.assertTrue; 22 | import static org.junit.Assert.fail; 23 | 24 | public class PutTest { 25 | RxDb db; 26 | RxLmdb lmdb; 27 | 28 | @Before 29 | public void before() { 30 | lmdb = RxLmdb.tmp(); 31 | db = lmdb.dbBuilder().build(); 32 | } 33 | 34 | @After 35 | public void after() { 36 | db.close(); 37 | db.lmdb.close(); 38 | } 39 | 40 | @Test(expected = NoSuchElementException.class) 41 | public void testPutAbort() { 42 | RxTx tx = lmdb.writeTx(); 43 | db.put(tx, Observable.from(_1_to_9)); 44 | tx.abort(); 45 | db.scan(KeyRange.forward()).toBlocking().first(); 46 | } 47 | 48 | @Test(expected = NoSuchElementException.class) 49 | public void testPutAbortWithResources() { 50 | try (RxTx tx = lmdb.writeTx()) { 51 | db.put(tx, Observable.from(_1_to_9)); 52 | } 53 | db.scan(KeyRange.forward()).toBlocking().first(); 54 | } 55 | 56 | @Test 57 | public void testPut() { 58 | db.put(Fixture.values[0]); 59 | assertArrayEquals(db.get(__1), __1); 60 | } 61 | 62 | @Test 63 | public void testPutCommit() { 64 | RxTx tx = lmdb.writeTx(); 65 | db.put(tx, Observable.from(_1_to_9)); 66 | tx.commit(); 67 | LinkedList expected = Fixture.range(__1, __9); 68 | toStreamBlocking(db.scan(KeyRange.forward())) 69 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 70 | assertTrue(expected.isEmpty()); 71 | } 72 | 73 | @Test 74 | public void testAppendCommit() { 75 | RxTx tx = lmdb.writeTx(); 76 | db.append(tx, Observable.from(_1_to_9)); 77 | tx.commit(); 78 | LinkedList expected = Fixture.range(__1, __9); 79 | toStreamBlocking(db.scan(KeyRange.forward())) 80 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 81 | assertTrue(expected.isEmpty()); 82 | } 83 | 84 | @Test(expected = NullPointerException.class) 85 | public void testPutException() throws InterruptedException { 86 | db.put(Observable.just((KeyValue) null)).toBlocking().first(); 87 | } 88 | 89 | @Test 90 | public void testPutOnErrorResumeNext() { 91 | AtomicReference t = new AtomicReference<>(); 92 | db.put(Observable.from(new KeyValue[]{values[0], null, values[2]})) 93 | .onErrorResumeNext(throwable -> { 94 | t.set(throwable); 95 | return Observable.just(false); 96 | }).subscribe(); 97 | LinkedList expected = Fixture.range(__1, __3); 98 | toStreamBlocking(db.scan()) 99 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 100 | assertThat(t.get()).isInstanceOf(NullPointerException.class); 101 | } 102 | 103 | @Test(expected = NoSuchElementException.class) 104 | public void testPutDoOnErrorAbort() throws InterruptedException { 105 | AtomicReference t = new AtomicReference<>(); 106 | RxTx tx = lmdb.writeTx(); 107 | try { 108 | db.put(tx, Observable.from(new KeyValue[]{values[0], null, values[2]})) 109 | .doOnError(throwable -> { 110 | t.set(throwable); 111 | tx.abort(); 112 | }).toBlocking().last(); 113 | fail("should throw npe"); 114 | } catch (NullPointerException e) { 115 | } 116 | db.scan(KeyRange.forward()).toBlocking().first(); 117 | } 118 | 119 | @Test 120 | public void testPutExceptionAsync() throws InterruptedException { 121 | AtomicReference t = new AtomicReference<>(); 122 | db.put(Observable.just((KeyValue) null)) 123 | .observeOn(Schedulers.io()) 124 | .doOnError(throwable -> t.set(throwable)) 125 | .subscribe(); 126 | Thread.sleep(100); 127 | LinkedList expected = Fixture.range(__1, __3); 128 | toStreamBlocking(db.scan()) 129 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 130 | assertThat(t.get()).isInstanceOf(NullPointerException.class); 131 | } 132 | 133 | @Test 134 | public void testPutAndCommitTxOnSeparateThreads() throws InterruptedException { 135 | RxTx tx = lmdb.writeTx(); 136 | Observable obs = Observable.from(_1_to_9) 137 | .subscribeOn(Schedulers.io()); 138 | db.put(tx, obs); 139 | Thread.sleep(200); 140 | tx.commit(); 141 | LinkedList expected = Fixture.range(__1, __9); 142 | toStreamBlocking(db.scan(KeyRange.forward())) 143 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 144 | assertThat(expected).isEqualTo(new LinkedList<>()); 145 | } 146 | 147 | @Test(expected = NoSuchElementException.class) 148 | public void testPutAndRollbackTxOnSeparateThreads() throws InterruptedException { 149 | RxTx tx = lmdb.writeTx(); 150 | Observable obs = Observable.from(_1_to_9) 151 | .subscribeOn(Schedulers.io()); 152 | db.put(tx, obs); 153 | Thread.sleep(200); 154 | tx.abort(); 155 | // aborted so NoSuchElementException is expected 156 | db.scan(KeyRange.forward()).toBlocking().first(); 157 | } 158 | 159 | @Test(expected = NoSuchElementException.class) 160 | public void testPutScanWithinTxThenAbort() { 161 | RxTx tx = lmdb.writeTx(); 162 | db.put(tx, Observable.from(_1_to_9)); 163 | LinkedList expected = Fixture.range(__1, __9); 164 | // should see values within same yet-to-commit tx 165 | toStreamBlocking(db.scan(tx, KeyRange.forward())) 166 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 167 | assertThat(expected).isEqualTo(new LinkedList<>()); 168 | tx.abort(); 169 | // aborted so NoSuchElementException is expected 170 | db.scan(KeyRange.forward()).toBlocking().first(); 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /jmh/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | org.deephacks.rxlmdb 6 | rxlmdb-project 7 | 0.0.5-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | rxlmdb-jmh 12 | rxlmdb-jmh 13 | jar 14 | 15 | 16 | 17 | com.squareup.wire 18 | wire-runtime 19 | 1.8.0 20 | 21 | 22 | org.deephacks.rxlmdb 23 | rxlmdb 24 | 0.0.5-SNAPSHOT 25 | 26 | 27 | org.deephacks.lmdbjni 28 | lmdbjni-linux64 29 | ${lmdbjni-version} 30 | 31 | 32 | uk.co.real-logic 33 | sbe 34 | 1.0.1-RC2 35 | 36 | 37 | org.deephacks.vals 38 | vals 39 | 0.6.1 40 | 41 | 42 | com.squareup 43 | javawriter 44 | 2.4.0 45 | 46 | 47 | org.openjdk.jmh 48 | jmh-core 49 | 1.10 50 | 51 | 52 | org.openjdk.jmh 53 | jmh-generator-annprocess 54 | 1.5 55 | 56 | 57 | org.slf4j 58 | slf4j-simple 59 | 1.7.12 60 | 61 | 62 | 63 | 64 | 65 | org.codehaus.mojo 66 | exec-maven-plugin 67 | 1.3.2 68 | 69 | 70 | generate-sources 71 | 72 | java 73 | 74 | 75 | 76 | 77 | 78 | uk.co.real-logic 79 | sbe 80 | 81 | uk.co.real_logic.sbe.SbeTool 82 | 83 | 84 | sbe.output.dir 85 | ${project.build.directory}/generated-sources/sbe 86 | 87 | 88 | 89 | ${project.basedir}/src/main/sbe/schema.xml 90 | 91 | ${project.build.directory}/generated-sources/sbe 92 | 93 | 94 | 95 | uk.co.real-logic 96 | sbe 97 | 1.0.3-RC2 98 | 99 | 100 | 101 | 102 | com.squareup.wire 103 | wire-maven-plugin 104 | 1.8.0 105 | 106 | 107 | generate-sources 108 | 109 | generate-sources 110 | 111 | 112 | 113 | user.proto 114 | 115 | 116 | 117 | 118 | 119 | 120 | org.codehaus.mojo 121 | build-helper-maven-plugin 122 | 1.8 123 | 124 | 125 | add-source 126 | generate-sources 127 | 128 | add-source 129 | 130 | 131 | 132 | ${basedir}/target/generated-sources/wire 133 | ${basedir}/target/generated-sources/annotations 134 | ${basedir}/target/generated-sources/sbe 135 | 136 | 137 | 138 | 139 | 140 | 141 | org.apache.maven.plugins 142 | maven-shade-plugin 143 | 2.2 144 | 145 | 146 | package 147 | 148 | shade 149 | 150 | 151 | benchmarks 152 | 153 | 154 | org.openjdk.jmh.Main 155 | 156 | 157 | 158 | 159 | 163 | *:* 164 | 165 | META-INF/*.SF 166 | META-INF/*.DSA 167 | META-INF/*.RSA 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /rxlmdb/src/test/java/org/deephacks/rxlmdb/ScanTest.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import rx.Observable; 7 | 8 | import java.util.LinkedList; 9 | import java.util.NoSuchElementException; 10 | 11 | import static com.google.common.truth.Truth.assertThat; 12 | import static org.deephacks.rxlmdb.Fixture.*; 13 | import static org.deephacks.rxlmdb.RxObservables.toStreamBlocking; 14 | 15 | public class ScanTest { 16 | RxDb db; 17 | 18 | @Before 19 | public void before() { 20 | db = RxDb.tmp(); 21 | db.put(Observable.from(_1_to_9)); 22 | } 23 | 24 | @After 25 | public void after() { 26 | db.close(); 27 | db.lmdb.close(); 28 | } 29 | 30 | @Test 31 | public void testScanRangeForward() { 32 | LinkedList expected = Fixture.range(__2, __3); 33 | toStreamBlocking(db.scan(KeyRange.range(_2, _3))) 34 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 35 | assertThat(expected).isEqualTo(new LinkedList<>()); 36 | } 37 | 38 | @Test 39 | public void testCount() { 40 | Integer count = db.scan() 41 | .flatMap(Observable::from) 42 | .count().toBlocking().first(); 43 | assertThat(count).isEqualTo(9); 44 | } 45 | 46 | @Test 47 | public void testScanSingleRange() { 48 | LinkedList expected = Fixture.range(__1, __1); 49 | toStreamBlocking(db.scan(KeyRange.range(_1, _1))) 50 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 51 | assertThat(expected).isEqualTo(new LinkedList<>()); 52 | } 53 | 54 | @Test 55 | public void testScanOnly() { 56 | byte[] startStop = new byte[] { (byte) 2 }; 57 | LinkedList expected = Fixture.range(__2, __2); 58 | toStreamBlocking(db.scan(KeyRange.range(startStop, startStop))) 59 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 60 | assertThat(expected).isEqualTo(new LinkedList<>()); 61 | } 62 | 63 | @Test 64 | public void testScanRangeBackward() { 65 | LinkedList expected = Fixture.range(__3, __2); 66 | toStreamBlocking(db.scan(KeyRange.range(_3, _2))) 67 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 68 | assertThat(expected).isEqualTo(new LinkedList<>()); 69 | } 70 | 71 | @Test 72 | public void testScanAtLeast() { 73 | LinkedList expected = Fixture.range(__3, __9); 74 | toStreamBlocking(db.scan(KeyRange.atLeast(_3))) 75 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 76 | assertThat(expected).isEqualTo(new LinkedList<>()); 77 | } 78 | 79 | @Test 80 | public void testScanAtLeastBackward() { 81 | LinkedList expected = Fixture.range(__3, __1); 82 | toStreamBlocking(db.scan(KeyRange.atLeastBackward(__3))) 83 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 84 | assertThat(expected).isEqualTo(new LinkedList<>()); 85 | } 86 | 87 | @Test 88 | public void testScanAtMost() { 89 | LinkedList expected = Fixture.range(__1, __4); 90 | toStreamBlocking(db.scan(KeyRange.atMost(_4))) 91 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 92 | assertThat(expected).isEqualTo(new LinkedList<>()); 93 | } 94 | 95 | @Test 96 | public void testScanAtMostBackward() { 97 | LinkedList expected = Fixture.range(__9, __4); 98 | toStreamBlocking(db.scan(KeyRange.atMostBackward(_4))) 99 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 100 | assertThat(expected).isEqualTo(new LinkedList<>()); 101 | } 102 | 103 | @Test 104 | public void testScanForward() { 105 | LinkedList expected = Fixture.range(__1, __9); 106 | toStreamBlocking(db.scan(KeyRange.forward())) 107 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 108 | assertThat(expected).isEqualTo(new LinkedList<>()); 109 | } 110 | 111 | @Test 112 | public void testScanForwardUnsubscribe() throws InterruptedException { 113 | LinkedList expected = Fixture.range(__1, __6); 114 | toStreamBlocking(db.scan(1, KeyRange.forward()).takeWhile(kvs -> { 115 | assertThat(kvs.size()).isEqualTo(1); 116 | byte[] key = expected.pollFirst().key(); 117 | assertThat(key).isEqualTo(kvs.get(0).key()); 118 | return key[0] < 6; 119 | })); 120 | assertThat(expected).isEqualTo(new LinkedList<>()); 121 | } 122 | 123 | @Test 124 | public void testScanForwardUnsubscribeBeforeBufferDrained() throws InterruptedException { 125 | LinkedList expected = Fixture.range(__1, __6); 126 | toStreamBlocking(db.scan(KeyRange.forward()).takeWhile(kvs -> { 127 | KeyValue kv = null; 128 | for (int i = 0; i < kvs.size(); i++) { 129 | kv = kvs.get(i); 130 | if (kv.key()[0] > 6) { 131 | return false; 132 | } 133 | assertThat(kv.key()).isEqualTo(expected.pollFirst().key()); 134 | } 135 | return true; 136 | })); 137 | assertThat(expected).isEqualTo(new LinkedList<>()); 138 | } 139 | 140 | @Test 141 | public void testScanBackward() { 142 | LinkedList expected = Fixture.range(__9, __1); 143 | toStreamBlocking(db.scan(KeyRange.backward())) 144 | .forEach(kv -> assertThat(expected.pollFirst().key()).isEqualTo(kv.key())); 145 | assertThat(expected).isEqualTo(new LinkedList<>()); 146 | } 147 | 148 | @Test 149 | public void testScanBackwardUnsubscribe() throws InterruptedException { 150 | LinkedList expected = Fixture.range(__9, __6); 151 | toStreamBlocking(db.scan(1, KeyRange.backward()).takeWhile(kvs -> { 152 | assertThat(kvs.size()).isEqualTo(1); 153 | byte[] key = expected.pollFirst().key(); 154 | assertThat(key).isEqualTo(kvs.get(0).key()); 155 | return key[0] > 6; 156 | })); 157 | assertThat(expected).isEqualTo(new LinkedList<>()); 158 | } 159 | 160 | @Test 161 | public void testScanBackwardUnsubscribeBeforeBufferDrained() throws InterruptedException { 162 | LinkedList expected = Fixture.range(__9, __6); 163 | toStreamBlocking(db.scan(KeyRange.backward()).takeWhile(kvs -> { 164 | KeyValue kv = null; 165 | for (int i = 0; i < kvs.size(); i++) { 166 | kv = kvs.get(i); 167 | if (kv.key()[0] < 6) { 168 | return false; 169 | } 170 | assertThat(kv.key()).isEqualTo(expected.pollFirst().key()); 171 | } 172 | return true; 173 | })); 174 | assertThat(expected).isEqualTo(new LinkedList<>()); 175 | } 176 | 177 | @Test(expected = NoSuchElementException.class) 178 | public void testScanEmptyRange() { 179 | db.scan(KeyRange.range(new byte[]{12}, new byte[]{123})).toBlocking().first(); 180 | } 181 | 182 | @Test 183 | public void testScanMapper() { 184 | LinkedList expected = Fixture.range(__1, __9); 185 | toStreamBlocking(db.scan((key, value) -> key.getByte(0))) 186 | .forEach(k -> assertThat(expected.pollFirst().key()[0]).isEqualTo(k)); 187 | assertThat(expected).isEqualTo(new LinkedList<>()); 188 | } 189 | 190 | @Test(expected = NoSuchElementException.class) 191 | public void testScanMapperNull() { 192 | db.scan((key, value) -> null).toBlocking().first(); 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | org.deephacks.rxlmdb 6 | rxlmdb-project 7 | rxlmdb-project 8 | 0.0.5-SNAPSHOT 9 | pom 10 | LMDB for RxJava 11 | http://rxlmdb.deephacks.org 12 | 2015 13 | 14 | 15 | 16 | Development List 17 | rxlmdb-dev@googlegroups.com 18 | 19 | 20 | User List 21 | rxlmdb-user@googlegroups.com 22 | 23 | 24 | 25 | 26 | github 27 | https://github.com/deephacks/RxLMDB/issues 28 | 29 | 30 | 31 | scm:git:git@github.com:deephacks/RxLMDB.git 32 | scm:git:git@github.com:deephacks/RxLMDB.git 33 | scm:git:git@github.com/deephacks/RxLMDB 34 | HEAD 35 | 36 | 37 | 38 | 39 | Apache License 40 | license.txt 41 | repo 42 | 43 | 44 | 45 | 46 | 47 | Kristoffer Sjogren 48 | krisskross 49 | stoffe -at- gmail.com 50 | 51 | 52 | Developer 53 | 54 | http://stoffe.deephacks.org/ 55 | +1 56 | 57 | 58 | 59 | 60 | 61 | jfrog 62 | https://oss.jfrog.org/libs-snapshot 63 | 64 | 65 | 66 | 67 | 0.4.6 68 | 0.2.2 69 | 1.0.0 70 | 0.0.1 71 | 0.0.1-20151126.205252-9 72 | UTF-8 73 | 74 | 75 | 76 | rxlmdb 77 | rxlmdb-grpc 78 | 79 | jmh 80 | 81 | 82 | 83 | 84 | 85 | maven-compiler-plugin 86 | org.apache.maven.plugins 87 | 2.3.2 88 | 89 | 1.8 90 | 1.8 91 | 1.8 92 | 93 | 94 | 95 | maven-deploy-plugin 96 | org.apache.maven.plugins 97 | 2.8.2 98 | 99 | 100 | maven-release-plugin 101 | 2.5.2 102 | 103 | false 104 | -P sign-artifacts 105 | true 106 | false 107 | @{project.version} 108 | 109 | 110 | 111 | maven-jar-plugin 112 | org.apache.maven.plugins 113 | 2.4 114 | 115 | 116 | 117 | ${project.name} 118 | ${project.version} 119 | deephacks 120 | ${project.name} 121 | ${project.version} 122 | deephacks 123 | org.deephacks 124 | http://rxlmdb.deephacks.org 125 | 126 | 127 | 128 | 129 | 130 | org.apache.maven.plugins 131 | maven-source-plugin 132 | 2.2.1 133 | 134 | 135 | attach-sources 136 | 137 | jar 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | junit 148 | junit 149 | 4.10 150 | test 151 | 152 | 153 | com.google.truth 154 | truth 155 | 0.26 156 | test 157 | 158 | 159 | org.deephacks.lmdbjni 160 | lmdbjni-linux64 161 | ${lmdbjni-version} 162 | test 163 | 164 | 165 | 166 | 167 | 168 | sign-artifacts 169 | 170 | 171 | 172 | maven-javadoc-plugin 173 | org.apache.maven.plugins 174 | 2.9.1 175 | 176 | 177 | attach-javadocs 178 | 179 | jar 180 | 181 | 182 | 183 | 184 | aggregate 185 | 186 | site 187 | 188 | 189 | 190 | false 191 | 192 | 193 | 194 | org.apache.maven.plugins 195 | maven-gpg-plugin 196 | 1.6 197 | 198 | 199 | sign-artifacts 200 | verify 201 | 202 | sign 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | sonatype-nexus-staging 214 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 215 | 216 | 217 | 218 | -------------------------------------------------------------------------------- /rxlmdb-grpc/src/main/java/org/deephacks/rxlmdb/RxDbGrpcClient.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import com.google.protobuf.ByteString; 4 | import com.google.protobuf.UnsafeByteStrings; 5 | import io.grpc.internal.ManagedChannelImpl; 6 | import io.grpc.netty.NegotiationType; 7 | import io.grpc.netty.NettyChannelBuilder; 8 | import io.grpc.stub.StreamObserver; 9 | import org.deephacks.rxlmdb.DatabaseServiceGrpc.DatabaseServiceStub; 10 | import org.deephacks.rxlmdb.Rxdb.*; 11 | import rx.Observable; 12 | import rx.Subscriber; 13 | 14 | import java.net.InetSocketAddress; 15 | import java.nio.ByteBuffer; 16 | import java.util.List; 17 | import java.util.Optional; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | public class RxDbGrpcClient { 21 | private final InetSocketAddress address; 22 | private final ManagedChannelImpl channel; 23 | private final DatabaseServiceStub stub; 24 | 25 | private RxDbGrpcClient(Builder builder) { 26 | String host = Optional.ofNullable(builder.host).orElse("localhost"); 27 | int port = Optional.ofNullable(builder.port).orElse(RxDbGrpcServer.DEFAULT_PORT); 28 | this.address = new InetSocketAddress(host, port); 29 | this.channel = NettyChannelBuilder.forAddress(this.address) 30 | .flowControlWindow(65 * 1024) 31 | .negotiationType(NegotiationType.PLAINTEXT) 32 | .build(); 33 | this.stub = DatabaseServiceGrpc.newStub(channel); 34 | } 35 | 36 | public Observable put(KeyValue kv) { 37 | if (kv == null || kv.key() == null || kv.key().length == 0) { 38 | return Observable.just(false); 39 | } 40 | return Observable.create(new Observable.OnSubscribe() { 41 | @Override 42 | public void call(Subscriber subscriber) { 43 | ByteString k = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(kv.key())); 44 | ByteString v = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(kv.value())); 45 | PutMsg put = PutMsg.newBuilder().setKey(k).setVal(v).build(); 46 | stub.put(put, new StreamObserver() { 47 | @Override 48 | public void onNext(Empty value) { 49 | } 50 | 51 | @Override 52 | public void onError(Throwable t) { 53 | subscriber.onError(t); 54 | } 55 | 56 | @Override 57 | public void onCompleted() { 58 | subscriber.onNext(true); 59 | subscriber.onCompleted(); 60 | } 61 | }); 62 | } 63 | }); 64 | } 65 | 66 | public void batch(Observable> values) { 67 | if (values == null) { 68 | return; 69 | } 70 | StreamObserver batch = stub.batch(new StreamObserver() { 71 | @Override 72 | public void onNext(Empty value) { 73 | 74 | } 75 | 76 | @Override 77 | public void onError(Throwable t) { 78 | 79 | } 80 | 81 | @Override 82 | public void onCompleted() { 83 | 84 | } 85 | }); 86 | 87 | values.subscribe(new Subscriber>() { 88 | @Override 89 | public void onCompleted() { 90 | batch.onCompleted(); 91 | } 92 | 93 | @Override 94 | public void onError(Throwable throwable) { 95 | batch.onError(throwable); 96 | } 97 | 98 | @Override 99 | public void onNext(List kvs) { 100 | for (KeyValue kv : kvs) { 101 | ByteString k = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(kv.key())); 102 | ByteString v = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(kv.value())); 103 | PutMsg put = PutMsg.newBuilder().setKey(k).setVal(v).build(); 104 | batch.onNext(put); 105 | } 106 | } 107 | }); 108 | } 109 | 110 | public Observable delete(byte[] key) { 111 | if (key == null || key.length == 0) { 112 | return Observable.just(false); 113 | } 114 | return Observable.create(new Observable.OnSubscribe() { 115 | @Override 116 | public void call(Subscriber subscriber) { 117 | ByteString k = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(key)); 118 | DeleteMsg delete = DeleteMsg.newBuilder().setKey(k).build(); 119 | stub.delete(delete, new StreamObserver() { 120 | @Override 121 | public void onNext(BooleanMsg value) { 122 | subscriber.onNext(value.getValue()); 123 | } 124 | 125 | @Override 126 | public void onError(Throwable t) { 127 | subscriber.onError(t); 128 | } 129 | 130 | @Override 131 | public void onCompleted() { 132 | subscriber.onCompleted(); 133 | } 134 | }); 135 | } 136 | }); 137 | } 138 | 139 | public Observable get(byte[] key) { 140 | if (key == null || key.length == 0) { 141 | return Observable.just(null); 142 | } 143 | return Observable.create(new Observable.OnSubscribe() { 144 | @Override 145 | public void call(Subscriber subscriber) { 146 | ByteString k = UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(key)); 147 | GetMsg get = GetMsg.newBuilder().setKey(k).build(); 148 | stub.get(get, new StreamObserver() { 149 | @Override 150 | public void onNext(ValueMsg value) { 151 | if (!value.getVal().isEmpty()) { 152 | KeyValue kv = new KeyValue(key, value.getVal().toByteArray()); 153 | subscriber.onNext(kv); 154 | } 155 | } 156 | 157 | @Override 158 | public void onError(Throwable t) { 159 | subscriber.onError(t); 160 | } 161 | 162 | @Override 163 | public void onCompleted() { 164 | subscriber.onCompleted(); 165 | } 166 | }); 167 | } 168 | }); 169 | } 170 | 171 | public Observable scan() { 172 | return scan(KeyRange.forward()); 173 | } 174 | 175 | public Observable scan(KeyRange range) { 176 | if (range == null) { 177 | return Observable.just(null); 178 | } 179 | return Observable.create(new Observable.OnSubscribe() { 180 | @Override 181 | public void call(Subscriber subscriber) { 182 | KeyRangeMsg.Builder builder = KeyRangeMsg.newBuilder(); 183 | if (range.start != null) { 184 | builder.setStart(ByteString.copyFrom(range.start)); 185 | } 186 | if (range.stop != null) { 187 | builder.setStop(ByteString.copyFrom(range.stop)); 188 | } 189 | builder.setTypeValue(range.type.ordinal()); 190 | stub.scan(builder.build(), new StreamObserver() { 191 | @Override 192 | public void onNext(KeyValueMsg msg) { 193 | KeyValue kv = new KeyValue(msg.getKey().toByteArray(), msg.getVal().toByteArray()); 194 | subscriber.onNext(kv); 195 | } 196 | 197 | @Override 198 | public void onError(Throwable t) { 199 | subscriber.onError(t); 200 | } 201 | 202 | @Override 203 | public void onCompleted() { 204 | subscriber.onCompleted(); 205 | } 206 | }); 207 | } 208 | }); 209 | } 210 | 211 | public void close() throws Exception { 212 | this.channel.shutdown(); 213 | this.channel.awaitTermination(10, TimeUnit.SECONDS); 214 | } 215 | 216 | public static Builder builder() { 217 | return new Builder(); 218 | } 219 | 220 | public static class Builder { 221 | private Integer port; 222 | private String host; 223 | 224 | public RxDbGrpcClient build() { 225 | return new RxDbGrpcClient(this); 226 | } 227 | 228 | public Builder host(String host) { 229 | this.host = host; 230 | return this; 231 | } 232 | 233 | public Builder port(int port) { 234 | this.port = port; 235 | return this; 236 | } 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # RxLMDB 2 | [![Build Status](https://travis-ci.org/deephacks/RxLMDB.svg?branch=master)](https://travis-ci.org/deephacks/RxLMDB) 3 | 4 | RxLMDB provide a [RxJava](https://github.com/ReactiveX/RxJava) API to [LMDB](http://symas.com/mdb/) (through [lmdbjni](https://github.com/deephacks/lmdbjni)) which is an ultra-fast, ultra-compact key-value embedded data store developed by Symas for the OpenLDAP Project. LMDB uses memory-mapped files, so it has the read performance of a pure in-memory database while still offering the persistence of standard disk-based databases. Transactional with full ACID semantics and crash-proof by design. No corruption. No startup time. Zero-config cache tuning. 5 | 6 | ### Why Rx + LMDB? 7 | 8 | Java 8 and RxJava is a pleasure to work with but since the LMDB API is a bit low level it make sense to raise the abstraction level to modern standards without scarifying too much (??) performance. So extending LMDB with RxJava makes it possible for asynchronous and event-based programs to process data from LMDB as sequences and adds operators that allow you to compose sequences together declaratively while abstracting away concerns about things like low-level threading, synchronization, thread-safety and concurrent data structures. 9 | 10 | RxLMDB provide a binary [gRPC](http://www.grpc.io) HTTP/2 interface to LMDB with capabilities such as bidirectional streaming, flow control, header compression, multiplexing requests over a single TCP connection. [ReactiveSocket](http://reactivesocket.io/) with [Aeron](https://github.com/real-logic/Aeron) are under evaluation as an alternative remote interface. 11 | 12 | ### Benchmark 13 | 14 | ##### Conclusion 15 | 16 | * If you want to run slow, copy-parse everything, like protobuf. 17 | * If you want to run fast, zero-copy-parse only what you need. 18 | * If you want to run faster, also use parallel range scans. 19 | * If you want to run fastest, do not use RxLMDB, but plain LMDB. 20 | * Better hardware obviously matters, zero-copy-parse scales well, copy-parse scales badly. 21 | 22 | ##### 3.16.0-4-amd64, Linux Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz 23 | 24 | 1 Thread 25 | 26 | ```bash 27 | Benchmark Mode Cnt Score Error Units 28 | BigKeyValueForwardRangeScan.plain thrpt 10 1178232.202 ± 81015.649 ops/s 29 | BigKeyValueForwardRangeScan.rx thrpt 10 1162131.060 ± 112057.128 ops/s 30 | BigZeroCopyForwardRangeScan.plain thrpt 10 9299859.225 ± 2529503.812 ops/s 31 | KeyValueForwardRangeScan.plain thrpt 10 6674117.744 ± 1067856.172 ops/s 32 | KeyValueForwardRangeScan.rx thrpt 10 5323064.014 ± 1061179.864 ops/s 33 | KeyValueForwardSkipRangeScan.plain thrpt 10 8789483.189 ± 768294.614 ops/s 34 | KeyValueForwardSkipRangeScan.rx thrpt 10 6453558.501 ± 903252.457 ops/s 35 | ProtoForwardRangeScan.plain thrpt 10 977556.340 ± 263740.090 ops/s 36 | ProtoForwardRangeScan.rx thrpt 10 842469.488 ± 170672.957 ops/s 37 | SbeForwardRangeScan.plain thrpt 10 5924733.706 ± 1985892.580 ops/s 38 | SbeForwardRangeScan.rx thrpt 10 4570195.110 ± 500547.365 ops/s 39 | ValsForwardRangeScan.plain thrpt 10 5365088.191 ± 2345685.548 ops/s 40 | ValsForwardRangeScan.rx thrpt 10 3627839.672 ± 1284540.222 ops/s 41 | ``` 42 | 43 | 4 Threads 44 | 45 | ```bash 46 | Benchmark Mode Cnt Score Error Units 47 | BigKeyValueForwardRangeScan.plain thrpt 10 1978242.823 ± 174190.990 ops/s 48 | BigKeyValueForwardRangeScan.rx thrpt 10 1699797.802 ± 147330.769 ops/s 49 | BigZeroCopyForwardRangeScan.plain thrpt 10 18631395.953 ± 7500005.892 ops/s 50 | KeyValueForwardRangeScan.plain thrpt 10 13384190.029 ± 2015610.137 ops/s 51 | KeyValueForwardRangeScan.rx thrpt 10 8646695.332 ± 2026413.388 ops/s 52 | KeyValueForwardSkipRangeScan.plain thrpt 10 14736089.587 ± 2432557.384 ops/s 53 | KeyValueForwardSkipRangeScan.rx thrpt 10 12330989.000 ± 559894.869 ops/s 54 | ProtoForwardRangeScan.plain thrpt 10 651203.480 ± 28715.405 ops/s 55 | ProtoForwardRangeScan.rx thrpt 10 617451.737 ± 20311.644 ops/s 56 | SbeForwardRangeScan.plain thrpt 10 8991860.431 ± 465302.254 ops/s 57 | SbeForwardRangeScan.rx thrpt 10 4755629.167 ± 1821428.568 ops/s 58 | ValsForwardRangeScan.plain thrpt 10 8546665.500 ± 1269468.808 ops/s 59 | ValsForwardRangeScan.rx thrpt 10 5812951.172 ± 573829.010 ops/s 60 | ``` 61 | 62 | ##### 3.16.0-4-amd64, Intel(R) Core(TM) i7-3740QM CPU @ 2.70GHz 63 | 64 | 8 threads 65 | 66 | ```bash 67 | Benchmark Mode Cnt Score Error Units 68 | BigKeyValueForwardRangeScan.plain thrpt 10 10933343.150 ± 89647.695 ops/s 69 | BigKeyValueForwardRangeScan.rx thrpt 10 9537755.671 ± 79734.499 ops/s 70 | BigZeroCopyForwardRangeScan.plain thrpt 10 67753109.315 ± 25043041.656 ops/s 71 | KeyValueForwardRangeScan.plain thrpt 10 51957281.758 ± 1315119.210 ops/s 72 | KeyValueForwardRangeScan.rx thrpt 10 37261517.010 ± 2339705.356 ops/s 73 | KeyValueForwardSkipRangeScan.plain thrpt 10 72329999.694 ± 9638355.993 ops/s 74 | KeyValueForwardSkipRangeScan.rx thrpt 10 49290830.102 ± 6230559.413 ops/s 75 | ProtoForwardRangeScan.plain thrpt 10 2043454.082 ± 96951.493 ops/s 76 | ProtoForwardRangeScan.rx thrpt 10 2129419.080 ± 199508.987 ops/s 77 | SbeForwardRangeScan.plain thrpt 10 59222391.194 ± 8513022.888 ops/s 78 | SbeForwardRangeScan.rx thrpt 10 43212267.029 ± 1891687.949 ops/s 79 | ValsForwardRangeScan.plain thrpt 10 54333422.372 ± 2917837.551 ops/s 80 | ValsForwardRangeScan.rx thrpt 10 39036264.187 ± 2346692.590 ops/s 81 | ``` 82 | 83 | 84 | ### Maven 85 | 86 | ```xml 87 | 88 | org.deephacks.rxlmdb 89 | rxlmdb 90 | ${rxlmdb.version} 91 | 92 | 93 | 94 | 95 | 96 | org.deephacks.lmdbjni 97 | lmdbjni-linux64 98 | ${lmdbjni.version} 99 | 100 | 101 | 102 | org.deephacks.lmdbjni 103 | lmdbjni-osx64 104 | ${lmdbjni.version} 105 | 106 | 107 | 108 | org.deephacks.lmdbjni 109 | lmdbjni-win64 110 | ${lmdbjni.version} 111 | 112 | 113 | 114 | org.deephacks.lmdbjni 115 | lmdbjni-android 116 | ${lmdbjni.version} 117 | 118 | ``` 119 | 120 | ### Usage 121 | 122 | ```java 123 | RxLmdb lmdb = RxLmdb.builder() 124 | .size(10, ByteUnit.GIBIBYTES) 125 | .path("/tmp/rxlmdb") 126 | .build(); 127 | 128 | RxDb db = lmdb.dbBuilder() 129 | .name("test") 130 | .build(); 131 | 132 | KeyValue[] kvs = new KeyValue[] { 133 | new KeyValue(new byte[] { 1 }, new byte[] { 1 }), 134 | new KeyValue(new byte[] { 2 }, new byte[] { 2 }), 135 | new KeyValue(new byte[] { 3 }, new byte[] { 3 }) 136 | }; 137 | 138 | // put 139 | db.put(Observable.from(kvs)); 140 | 141 | // get 142 | Observable o = db.get(Observable.just(new byte[] { 1 })); 143 | 144 | // RxJava have a hard time coping with extreme scan performance of LMDB without buffering, 145 | // hence the Observable list return value from scan operations. Just flatmap away and be happy. 146 | 147 | // scan forward 148 | Observable> o = db.scan(); 149 | 150 | // scan backward 151 | Observable> o = db.scan(KeyRange.backward()); 152 | 153 | // scan range forward 154 | Observable> o = db.scan( 155 | KeyRange.range(new byte[]{ 1 }, new byte[]{ 2 } 156 | ); 157 | 158 | // scan range backward 159 | Observable> o = db.scan( 160 | KeyRange.range(new byte[]{ 2 }, new byte[]{ 1 } 161 | ); 162 | 163 | // parallel range scans 164 | Observable> obs = db.scan( 165 | KeyRange.range(new byte[]{ 1 }, new byte[]{ 1 }), 166 | KeyRange.range(new byte[]{ 2 }, new byte[]{ 2 }), 167 | KeyRange.range(new byte[]{ 3 }, new byte[]{ 3 }) 168 | ); 169 | 170 | // zero copy parallel range scans 171 | Observable> obs = db.scan( 172 | (key, value) -> key.getByte(0), 173 | KeyRange.range(new byte[]{ 1 }, new byte[]{ 1 }), 174 | KeyRange.range(new byte[]{ 2 }, new byte[]{ 2 }), 175 | KeyRange.range(new byte[]{ 3 }, new byte[]{ 3 })); 176 | 177 | // Cursor scans 178 | Observable> obs = db.cursor((cursor, subscriber) -> { 179 | cursor.first(); 180 | subscriber.onNext(cursor.keyBytes()); 181 | cursor.last(); 182 | subscriber.onNext(cursor.keyBytes()); 183 | }); 184 | 185 | // count rows 186 | Integer count = db.scan() 187 | .flatMap(Observable::from) 188 | .count().toBlocking().first(); 189 | 190 | // delete 191 | db.delete(Observable.just(new byte[] { 1 })); 192 | 193 | // delete range 194 | Observable keys = db.scan() 195 | .flatMap(Observable::from) 196 | .map(kv -> kv.key); 197 | db.delete(keys); 198 | 199 | ``` 200 | 201 | The write amplification of LMDB's copy-on-write approach can sometimes become expensive. So for higher throughput, infinite data streams, RxLMDB provide effecient and asynchronous batching. Remember to use a ```SerializedSubject``` if multiple threads are writing concurrently. 202 | 203 | ```java 204 | SerializedSubject subject = PublishSubject.create().toSerialized(); 205 | db.batch(subject.buffer(10, TimeUnit.NANOSECONDS, 512)); 206 | subject.onNext(new KeyValue(new byte[] { 1 }, new byte[] { 1 })); 207 | subject.onNext(new KeyValue(new byte[] { 2 }, new byte[] { 2 })); 208 | subject.onCompleted(); 209 | ``` 210 | 211 | ### gRPC 212 | 213 | The gRPC interface is wrapped by a RxJava facade that mimic the RxLMDB API. 214 | 215 | ```java 216 | 217 | // server side 218 | 219 | RxLmdb lmdb = RxLmdb.builder() 220 | .size(10, ByteUnit.GIBIBYTES) 221 | .path("/tmp/rxlmdb") 222 | .build(); 223 | 224 | RxDb db = lmdb.dbBuilder() 225 | .name("test") 226 | .build(); 227 | 228 | RxDbGrpcServer server = RxDbGrpcServer.builder() 229 | .host("localhost").port(18080) 230 | .lmdb(lmdb).db(db) 231 | .build(); 232 | 233 | // client side 234 | 235 | RxDbGrpcClient client = RxDbGrpcClient.builder() 236 | .host("localhost").port(18080) 237 | .build(); 238 | 239 | Observable put = client.put(new KeyValue(new byte[1], new byte[1])); 240 | 241 | Observable get = client.get(new byte[1]); 242 | 243 | Observable delete = client.delete(new byte[1]); 244 | 245 | Observable forwardScan = client.scan(KeyRange.forward()); 246 | 247 | Observable backwardScan = client.scan(KeyRange.backward()); 248 | 249 | Observable rangeScan = client.scan(KeyRange.range(new byte[]{ 1 }, new byte[]{ 10 })); 250 | ``` 251 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/Scanners.java: -------------------------------------------------------------------------------- 1 | package org.deephacks.rxlmdb; 2 | 3 | import org.fusesource.lmdbjni.BufferCursor; 4 | import org.fusesource.lmdbjni.Database; 5 | import org.fusesource.lmdbjni.DirectBuffer; 6 | import rx.Observable; 7 | import rx.Scheduler; 8 | import rx.Subscriber; 9 | 10 | import java.util.Arrays; 11 | import java.util.List; 12 | 13 | import static org.deephacks.rxlmdb.DirectBufferComparator.compareTo; 14 | 15 | class Scanners { 16 | 17 | static final Observable> scan(Database db, 18 | RxTx tx, 19 | DirectMapper mapper, 20 | Scheduler scheduler, 21 | int buffer, 22 | KeyRange... ranges) { 23 | if (ranges.length == 0) { 24 | Scanner scanner = getScanner(db, tx, mapper, KeyRange.forward()); 25 | return createObservable(scanner, tx).buffer(buffer); 26 | } else if (ranges.length == 1) { 27 | Scanner scanner = getScanner(db, tx, mapper, ranges[0]); 28 | return createObservable(scanner, tx).buffer(buffer); 29 | } 30 | if (!tx.isUserManaged) { 31 | throw new IllegalArgumentException("Parallel scan transactions must be handled by the user"); 32 | } 33 | return Arrays.asList(ranges).stream() 34 | .map(range -> createObservable(getScanner(db, tx, mapper, range), tx) 35 | .buffer(buffer).subscribeOn(scheduler).onBackpressureBuffer()) 36 | .reduce(Observable.empty(), (o1, o2) -> o1.mergeWith(o2)); 37 | } 38 | 39 | static Observable> scan(Database db, int buffer, RxTx tx, CursorScanner scanner) { 40 | return createObservable(new CursorScan(db, tx, null, null, scanner), tx).buffer(buffer); 41 | } 42 | 43 | 44 | private static final Scanner getScanner(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 45 | switch (range.type) { 46 | case FORWARD: 47 | return new ForwardScan<>(db, tx, mapper, range); 48 | case FORWARD_START: 49 | return new ForwardStartScan<>(db, tx, mapper, range); 50 | case FORWARD_STOP: 51 | return new ForwardStopScan<>(db, tx, mapper, range); 52 | case FOWARD_RANGE: 53 | return new ForwardRangeScan<>(db, tx, mapper, range); 54 | case BACKWARD: 55 | return new BackwardScan<>(db, tx, mapper, range); 56 | case BACKWARD_START: 57 | return new BackwardStartScan<>(db, tx, mapper, range); 58 | case BACKWARD_STOP: 59 | return new BackwardStopScan<>(db, tx, mapper, range); 60 | case BACKWARD_RANGE: 61 | return new BackwardRangeScan<>(db, tx, mapper, range); 62 | default: 63 | return new ForwardScan<>(db, tx, mapper, range); 64 | } 65 | } 66 | 67 | private static final Observable createObservable(Scanner scanner, RxTx tx) { 68 | return Observable.create(subscriber -> { 69 | try { 70 | try { 71 | scanner.execute(subscriber); 72 | if (!subscriber.isUnsubscribed()) { 73 | subscriber.onCompleted(); 74 | } 75 | } catch (Throwable e) { 76 | if (scanner.cursor != null) { 77 | scanner.cursor.close(); 78 | } 79 | if (!tx.isUserManaged) { 80 | tx.abort(); 81 | } 82 | subscriber.onError(e); 83 | } finally { 84 | if (scanner.cursor != null) { 85 | scanner.cursor.close(); 86 | } 87 | if (!tx.isUserManaged) { 88 | // no op if tx was aborted 89 | tx.commit(); 90 | } 91 | } 92 | } catch (Exception e) { 93 | subscriber.onError(e); 94 | } 95 | }); 96 | 97 | } 98 | 99 | static abstract class Scanner { 100 | final Database db; 101 | final RxTx tx; 102 | final DirectMapper mapper; 103 | final KeyRange range; 104 | BufferCursor cursor; 105 | 106 | protected Scanner(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 107 | this.db = db; 108 | this.tx = tx; 109 | this.mapper = mapper; 110 | this.range = range; 111 | } 112 | 113 | public abstract void execute(Subscriber subscriber); 114 | } 115 | 116 | static class ForwardScan extends Scanner { 117 | 118 | protected ForwardScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 119 | super(db, tx, mapper, range); 120 | } 121 | 122 | @Override 123 | public void execute(Subscriber subscriber) { 124 | BufferCursor cursor = db.bufferCursor(tx.tx); 125 | boolean hasNext = cursor.first(); 126 | while (hasNext) { 127 | if (subscriber.isUnsubscribed()) { 128 | return; 129 | } 130 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 131 | if (result != null) { 132 | subscriber.onNext(result); 133 | } 134 | hasNext = cursor.next(); 135 | } 136 | } 137 | } 138 | 139 | static class BackwardScan extends Scanner { 140 | 141 | protected BackwardScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 142 | super(db, tx, mapper, range); 143 | } 144 | 145 | @Override 146 | public void execute(Subscriber subscriber) { 147 | BufferCursor cursor = db.bufferCursor(tx.tx); 148 | boolean hasNext = cursor.last(); 149 | while (hasNext) { 150 | if (subscriber.isUnsubscribed()) { 151 | return; 152 | } 153 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 154 | if (result != null) { 155 | subscriber.onNext(result); 156 | } 157 | hasNext = cursor.prev(); 158 | } 159 | } 160 | } 161 | 162 | static class ForwardStopScan extends Scanner { 163 | 164 | protected ForwardStopScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 165 | super(db, tx, mapper, range); 166 | } 167 | 168 | @Override 169 | public void execute(Subscriber subscriber) { 170 | BufferCursor cursor = db.bufferCursor(tx.tx); 171 | DirectBuffer stop = new DirectBuffer(range.stop); 172 | boolean hasNext = cursor.first(); 173 | while (hasNext) { 174 | if (compareTo(cursor.keyBuffer(), stop) <= 0) { 175 | if (subscriber.isUnsubscribed()) { 176 | return; 177 | } 178 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 179 | if (result != null) { 180 | subscriber.onNext(result); 181 | } 182 | } 183 | hasNext = cursor.next(); 184 | } 185 | } 186 | } 187 | 188 | static class BackwardStopScan extends Scanner { 189 | 190 | protected BackwardStopScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 191 | super(db, tx, mapper, range); 192 | } 193 | 194 | @Override 195 | public void execute(Subscriber subscriber) { 196 | BufferCursor cursor = db.bufferCursor(tx.tx); 197 | DirectBuffer stop = new DirectBuffer(range.stop); 198 | boolean hasNext = cursor.last(); 199 | while (hasNext) { 200 | if (subscriber.isUnsubscribed()) { 201 | return; 202 | } 203 | if (compareTo(cursor.keyBuffer(), stop) >= 0) { 204 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 205 | if (result != null) { 206 | subscriber.onNext(result); 207 | } 208 | } 209 | hasNext = cursor.prev(); 210 | } 211 | } 212 | } 213 | 214 | static class ForwardStartScan extends Scanner { 215 | 216 | protected ForwardStartScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 217 | super(db, tx, mapper, range); 218 | } 219 | 220 | @Override 221 | public void execute(Subscriber subscriber) { 222 | BufferCursor cursor = db.bufferCursor(tx.tx); 223 | boolean hasNext = cursor.seek(range.start); 224 | while (hasNext) { 225 | if (subscriber.isUnsubscribed()) { 226 | return; 227 | } 228 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 229 | if (result != null) { 230 | subscriber.onNext(result); 231 | } 232 | hasNext = cursor.next(); 233 | } 234 | } 235 | } 236 | 237 | static class BackwardStartScan extends Scanner { 238 | 239 | protected BackwardStartScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 240 | super(db, tx, mapper, range); 241 | } 242 | 243 | @Override 244 | public void execute(Subscriber subscriber) { 245 | BufferCursor cursor = db.bufferCursor(tx.tx); 246 | boolean hasNext = cursor.seek(range.start); 247 | while (hasNext) { 248 | if (subscriber.isUnsubscribed()) { 249 | return; 250 | } 251 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 252 | if (result != null) { 253 | subscriber.onNext(result); 254 | } 255 | hasNext = cursor.prev(); 256 | } 257 | } 258 | } 259 | 260 | 261 | static class ForwardRangeScan extends Scanner { 262 | 263 | protected ForwardRangeScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 264 | super(db, tx, mapper, range); 265 | } 266 | 267 | @Override 268 | public void execute(Subscriber subscriber) { 269 | BufferCursor cursor = db.bufferCursor(tx.tx); 270 | DirectBuffer stop = new DirectBuffer(range.stop); 271 | boolean hasNext = cursor.seek(range.start); 272 | if (!hasNext) { 273 | return; 274 | } else { 275 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 276 | if (result != null) { 277 | subscriber.onNext(result); 278 | } 279 | } 280 | while (!subscriber.isUnsubscribed() && cursor.next() && compareTo(cursor.keyBuffer(), stop) <= 0) { 281 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 282 | if (result != null) { 283 | subscriber.onNext(result); 284 | } 285 | } 286 | } 287 | } 288 | 289 | static class BackwardRangeScan extends Scanner { 290 | 291 | protected BackwardRangeScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range) { 292 | super(db, tx, mapper, range); 293 | } 294 | 295 | @Override 296 | public void execute(Subscriber subscriber) { 297 | BufferCursor cursor = db.bufferCursor(tx.tx); 298 | DirectBuffer stop = new DirectBuffer(range.stop); 299 | boolean hasNext = cursor.seek(range.start); 300 | while (hasNext) { 301 | if (subscriber.isUnsubscribed()) { 302 | return; 303 | } else if (compareTo(cursor.keyBuffer(), stop) >= 0) { 304 | T result = mapper.map(cursor.keyBuffer(), cursor.valBuffer()); 305 | if (result != null) { 306 | subscriber.onNext(result); 307 | } 308 | } 309 | hasNext = cursor.prev(); 310 | } 311 | } 312 | } 313 | 314 | static class CursorScan extends Scanner { 315 | private final CursorScanner scanner; 316 | 317 | protected CursorScan(Database db, RxTx tx, DirectMapper mapper, KeyRange range, CursorScanner scanner) { 318 | super(db, tx, mapper, range); 319 | this.scanner = scanner; 320 | } 321 | 322 | @Override 323 | public void execute(Subscriber subscriber) { 324 | try (BufferCursor cursor = db.bufferCursor(tx.tx)) { 325 | scanner.execute(cursor, subscriber); 326 | } 327 | } 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /rxlmdb/src/main/java/org/deephacks/rxlmdb/RxDb.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | *

6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | *

8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.deephacks.rxlmdb; 15 | 16 | import org.fusesource.lmdbjni.*; 17 | import rx.Observable; 18 | import rx.Scheduler; 19 | import rx.Subscriber; 20 | import rx.exceptions.OnErrorFailedException; 21 | import rx.subjects.PublishSubject; 22 | import rx.subjects.SerializedSubject; 23 | 24 | import java.nio.ByteBuffer; 25 | import java.util.List; 26 | import java.util.Optional; 27 | 28 | public class RxDb { 29 | /** copy by default */ 30 | final DirectMapper.KeyValueMapper KV_MAPPER = new DirectMapper.KeyValueMapper(); 31 | final RxLmdb lmdb; 32 | final Database db; 33 | final String name; 34 | final Scheduler scheduler; 35 | final int defaultBuffer = 512; 36 | 37 | private RxDb(Builder builder) { 38 | this.lmdb = builder.lmdb; 39 | this.name = Optional.ofNullable(builder.name).orElse("default"); 40 | this.db = lmdb.env.openDatabase(this.name); 41 | this.scheduler = lmdb.scheduler; 42 | } 43 | 44 | /** 45 | * Put kvs into the database and commit when the last 46 | * element has been written. 47 | */ 48 | public Observable put(Observable values) { 49 | return put(lmdb.internalWriteTx(), values); 50 | } 51 | 52 | public Boolean put(KeyValue kv) { 53 | db.put(kv.key(), kv.value()); 54 | return true; 55 | } 56 | 57 | public Boolean put(RxTx tx, KeyValue kv) { 58 | db.put(tx.tx, kv.key(), kv.value()); 59 | return true; 60 | } 61 | 62 | /** 63 | * Same as regular put but the user is in charge of the transaction. 64 | * 65 | * @see RxDb#put(Observable) 66 | */ 67 | public Observable put(RxTx tx, Observable values) { 68 | return put(tx, values, false); 69 | } 70 | 71 | /** 72 | * @see RxDb#append(RxTx, Observable) 73 | */ 74 | public void append(Observable values) { 75 | append(lmdb.internalWriteTx(), values); 76 | } 77 | 78 | /** 79 | * Append the kvs to the end of the database without comparing its order first. 80 | * Appending a key that is not greater than the highest existing key will 81 | * cause corruption. 82 | */ 83 | public void append(RxTx tx, Observable values) { 84 | put(tx, values, true); 85 | } 86 | 87 | public void append(RxTx tx, KeyValue kv) { 88 | db.put(tx.tx, kv.key(), kv.value(), Constants.APPEND); 89 | } 90 | 91 | public void append(KeyValue kv) { 92 | db.put(kv.key(), kv.value(), Constants.APPEND); 93 | } 94 | 95 | /** 96 | * Write and commit kvs asynchronously in batches. The user is free to 97 | * use choose whatever buffering configuration is needed. 98 | */ 99 | public void batch(Observable> values) { 100 | BatchSubscriber putSubscriber = new BatchSubscriber(this); 101 | values.subscribe(putSubscriber); 102 | } 103 | 104 | private Observable put(RxTx tx, Observable values, boolean append) { 105 | PutSubscriber putSubscriber = new PutSubscriber(this, tx, append); 106 | values.subscribe(putSubscriber); 107 | return putSubscriber.result; 108 | } 109 | 110 | /** 111 | * Get kvs from the database. Items that are not found will be 112 | * represented as null. 113 | */ 114 | public Observable get(Observable keys) { 115 | return get(lmdb.internalReadTx(), KV_MAPPER, keys); 116 | } 117 | 118 | /** 119 | * @see RxDb#get(Observable) 120 | */ 121 | public Observable get(RxTx tx, Observable keys) { 122 | return get(tx, KV_MAPPER, keys); 123 | } 124 | 125 | /** 126 | * Allow zero copy transformation of the resulting values. 127 | * 128 | * @see RxDb#get(Observable) 129 | */ 130 | public Observable get(DirectMapper mapper, Observable keys) { 131 | return get(lmdb.internalReadTx(), mapper, keys); 132 | } 133 | 134 | /** 135 | * Allow zero copy transformation of the resulting values. 136 | * 137 | * @see RxDb#get(Observable) 138 | */ 139 | public Observable get(RxTx tx, DirectMapper mapper, Observable keys) { 140 | return keys.flatMap(key -> { 141 | try { 142 | ByteBuffer bb = ByteBuffer.allocateDirect(key.length).put(key); 143 | DirectBuffer keyBuffer = new DirectBuffer(bb); 144 | DirectBuffer valBuffer = new DirectBuffer(0, 0); 145 | if (LMDBException.NOTFOUND != db.get(tx.tx, keyBuffer, valBuffer)) { 146 | return Observable.just(mapper.map(keyBuffer, valBuffer)); 147 | } else { 148 | return Observable.just(null); 149 | } 150 | } catch (Throwable e) { 151 | return Observable.just(null); 152 | } 153 | }).doOnCompleted(() -> { 154 | if (!tx.isUserManaged) { 155 | tx.commit(); 156 | } 157 | }); 158 | } 159 | 160 | public byte[] get(byte[] key) { 161 | return db.get(key); 162 | } 163 | 164 | public byte[] get(RxTx tx, byte[] key) { 165 | return db.get(tx.tx, key); 166 | } 167 | 168 | /** 169 | * @see RxDb#delete(RxTx) 170 | */ 171 | public void delete() { 172 | delete(lmdb.internalWriteTx()); 173 | } 174 | 175 | /** 176 | * Delete all records 177 | */ 178 | public void delete(RxTx tx) { 179 | // non-blocking needed? 180 | scan(tx).toBlocking().forEach(keyValues -> { 181 | Observable keys = keyValues.stream() 182 | .map(kv -> Observable.just(kv.key())) 183 | .reduce(Observable.empty(), (o1, o2) -> o1.mergeWith(o2)); 184 | delete(tx, keys); 185 | }); 186 | } 187 | 188 | /** 189 | * @see RxDb#delete(RxTx, Observable) 190 | */ 191 | public void delete(Observable keys) { 192 | delete(lmdb.internalWriteTx(), keys); 193 | } 194 | 195 | /** 196 | * Delete records associated with provided keys. 197 | */ 198 | public void delete(RxTx tx, Observable keys) { 199 | keys.subscribe(new DeleteSubscriber(this, tx)); 200 | } 201 | 202 | public Observable> scan(DirectMapper scan) { 203 | return scan(defaultBuffer, lmdb.internalReadTx(), scan); 204 | } 205 | 206 | public boolean delete(byte[] key) { 207 | return db.delete(key); 208 | } 209 | 210 | public boolean delete(RxTx tx, byte[] key) { 211 | return db.delete(tx.tx, key); 212 | } 213 | 214 | /** 215 | * Forward scan the database with a configurable buffer size and the 216 | * ability to zero copy transformation of the resulting values. 217 | */ 218 | public Observable> scan(int buffer, DirectMapper mapper) { 219 | return scan(buffer, lmdb.internalReadTx(), mapper); 220 | } 221 | 222 | /** 223 | * Scan multiple ranges of kvs in parallel. 224 | */ 225 | public Observable> scan(KeyRange... ranges) { 226 | return scan(defaultBuffer, lmdb.internalReadTx(), KV_MAPPER, ranges); 227 | } 228 | 229 | public Observable> scan(int buffer, KeyRange... ranges) { 230 | return scan(buffer, lmdb.internalReadTx(), KV_MAPPER, ranges); 231 | } 232 | 233 | public Observable> scan(RxTx tx, KeyRange... ranges) { 234 | return scan(defaultBuffer, tx, KV_MAPPER, ranges); 235 | } 236 | 237 | public Observable> scan(int buffer, RxTx tx, KeyRange... ranges) { 238 | return scan(buffer, tx, KV_MAPPER, ranges); 239 | } 240 | 241 | public Observable> scan(DirectMapper mapper, KeyRange... ranges) { 242 | return scan(defaultBuffer, lmdb.internalReadTx(), mapper, ranges); 243 | } 244 | 245 | public Observable> scan(int buffer, DirectMapper mapper, KeyRange... ranges) { 246 | return scan(buffer, lmdb.internalReadTx(), mapper, ranges); 247 | } 248 | 249 | public Observable> scan(RxTx tx, DirectMapper mapper, KeyRange... ranges) { 250 | return scan(defaultBuffer, tx, mapper, ranges); 251 | } 252 | 253 | public Observable> scan(int buffer, RxTx tx, DirectMapper mapper, KeyRange... ranges) { 254 | return Scanners.scan(db, tx, mapper, scheduler, buffer, ranges); 255 | } 256 | 257 | public Observable> cursor(CursorScanner scanner) { 258 | return cursor(defaultBuffer, lmdb.internalReadTx(), scanner); 259 | } 260 | 261 | public Observable> cursor(RxTx tx, CursorScanner scanner) { 262 | return cursor(defaultBuffer, tx, scanner); 263 | } 264 | 265 | public Observable> cursor(int buffer, CursorScanner scanner) { 266 | return cursor(buffer, lmdb.internalReadTx(), scanner); 267 | } 268 | 269 | public Observable> cursor(int buffer, RxTx tx, CursorScanner scanner) { 270 | return Scanners.scan(db, buffer, tx, scanner); 271 | } 272 | 273 | public static Builder builder() { 274 | return new Builder(); 275 | } 276 | 277 | public static RxDb tmp() { 278 | return new Builder().lmdb(RxLmdb.tmp()).build(); 279 | } 280 | 281 | public String getName() { 282 | return name; 283 | } 284 | 285 | public long getSize() { 286 | return lmdb.env.info().getMapSize(); 287 | } 288 | 289 | public void close() { 290 | db.close(); 291 | } 292 | 293 | public static class Builder { 294 | private String name; 295 | private RxLmdb lmdb; 296 | 297 | public Builder lmdb(RxLmdb lmdb) { 298 | this.lmdb = lmdb; 299 | return this; 300 | } 301 | 302 | public Builder name(String name) { 303 | this.name = name; 304 | return this; 305 | } 306 | 307 | public RxDb build() { 308 | return new RxDb(this); 309 | } 310 | } 311 | 312 | private static class PutSubscriber extends Subscriber implements Loggable { 313 | final RxTx tx; 314 | final Database db; 315 | final boolean append; 316 | final PublishSubject result; 317 | 318 | private PutSubscriber(RxDb db, RxTx tx, boolean append) { 319 | this.tx = tx; 320 | this.db = db.db; 321 | this.append = append; 322 | this.result = PublishSubject.create(); 323 | } 324 | 325 | @Override 326 | public void onCompleted() { 327 | if (!tx.isUserManaged) { 328 | tx.commit(); 329 | } 330 | result.onCompleted(); 331 | } 332 | 333 | @Override 334 | public void onError(Throwable e) { 335 | tx.abort(); 336 | logger().error("Put error.", e); 337 | result.onError(e); 338 | } 339 | 340 | @Override 341 | public void onNext(KeyValue kv) { 342 | try { 343 | db.put(tx.tx, kv.key(), kv.value(), append ? Constants.APPEND : 0); 344 | result.onNext(true); 345 | } catch (Throwable e) { 346 | if (e instanceof RuntimeException) { 347 | throw e; 348 | } 349 | throw new OnErrorFailedException(e); 350 | } 351 | } 352 | } 353 | 354 | private static class BatchSubscriber extends Subscriber> implements Loggable { 355 | final Database db; 356 | final Env env; 357 | 358 | private BatchSubscriber(RxDb db) { 359 | this.env = db.lmdb.env; 360 | this.db = db.db; 361 | } 362 | 363 | @Override 364 | public void onCompleted() { 365 | } 366 | 367 | @Override 368 | public void onError(Throwable e) { 369 | logger().error("Batch error.", e); 370 | } 371 | 372 | @Override 373 | public void onNext(List kvs) { 374 | try { 375 | if (kvs.size() < 1) { 376 | return; 377 | } 378 | try (Transaction tx = env.createWriteTransaction()) { 379 | for (KeyValue kv : kvs) { 380 | try { 381 | db.put(tx, kv.key(), kv.value(), 0); 382 | } catch (Throwable e) { 383 | // log error, swallow exception and proceed to next kv 384 | logger().error("Batch put error.", e); 385 | } 386 | } 387 | tx.commit(); 388 | } 389 | } catch (Throwable e) { 390 | throw new OnErrorFailedException(e); 391 | } 392 | } 393 | } 394 | 395 | private static class DeleteSubscriber extends Subscriber implements Loggable { 396 | final RxTx tx; 397 | final Database db; 398 | 399 | private DeleteSubscriber(RxDb db, RxTx tx) { 400 | this.tx = tx; 401 | this.db = db.db; 402 | } 403 | 404 | @Override 405 | public void onCompleted() { 406 | if (!tx.isUserManaged) { 407 | tx.commit(); 408 | } 409 | } 410 | 411 | @Override 412 | public void onError(Throwable e) { 413 | logger().error("Delete error.", e); 414 | if (!tx.isUserManaged) { 415 | tx.abort(); 416 | } 417 | } 418 | 419 | @Override 420 | public void onNext(byte[] key) { 421 | db.delete(tx.tx, key); 422 | } 423 | } 424 | } 425 | --------------------------------------------------------------------------------