├── .gitignore ├── .travis.yml ├── bin ├── mintds-stop.sh ├── mintds-start.sh ├── mintds-cli.sh └── mintds-run-class.sh ├── src ├── main │ ├── java │ │ └── com │ │ │ └── arturmkrtchyan │ │ │ └── mintds │ │ │ ├── config │ │ │ ├── CountMinSketchConfig.java │ │ │ ├── BloomFilterConfig.java │ │ │ ├── ServerConfig.java │ │ │ └── Configuration.java │ │ │ ├── server │ │ │ ├── MintDsConfiguration.java │ │ │ ├── MintDsServer.java │ │ │ ├── netty │ │ │ │ ├── ResponseEncoder.java │ │ │ │ ├── RequestDecoder.java │ │ │ │ ├── NettyExceptionHandler.java │ │ │ │ ├── NettyServerHandler.java │ │ │ │ ├── NettyServer.java │ │ │ │ └── NettyServerInitializer.java │ │ │ └── MintDsDaemon.java │ │ │ ├── core │ │ │ ├── HyperLogLogStore.java │ │ │ ├── BloomFilterStore.java │ │ │ ├── CountMinSketchStore.java │ │ │ ├── KeyValueStore.java │ │ │ ├── KeyValueStoreRouter.java │ │ │ ├── CountingBloomFilterStore.java │ │ │ ├── AbstractKeyValueStore.java │ │ │ └── datastructure │ │ │ │ └── CountingBloomFilter.java │ │ │ └── cli │ │ │ └── MintDsTerminal.java │ ├── assembly │ │ ├── jar.xml │ │ └── dist.xml │ └── resources │ │ └── logback.xml └── test │ ├── resources │ └── logback-test.xml │ └── java │ └── com │ └── arturmkrtchyan │ └── mintds │ └── integration │ ├── AbstractKeyValueStoreIT.java │ ├── HyperLogLogIT.java │ ├── BloomFilterIT.java │ ├── CountMinSktechIT.java │ ├── MultithreadedAccessIT.java │ └── CountingBloomFilterIT.java ├── conf └── mintds.yaml ├── README.md ├── pom.xml └── LICENSE.txt /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | *.iml 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | after_success: 5 | - mvn clean verify 6 | -------------------------------------------------------------------------------- /bin/mintds-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ps ax | grep -i 'MintDsDaemon' | grep java | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/config/CountMinSketchConfig.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.config; 2 | 3 | public class CountMinSketchConfig { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/MintDsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server; 2 | 3 | public class MintDsConfiguration { 4 | // https://dzone.com/articles/using-yaml-java-application 5 | } 6 | -------------------------------------------------------------------------------- /bin/mintds-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# -lt 1 ]; 4 | then 5 | echo "USAGE: $0 conf/mintds.yaml" 6 | exit 1 7 | fi 8 | 9 | base_dir=$(dirname $0) 10 | 11 | exec $base_dir/mintds-run-class.sh com.arturmkrtchyan.mintds.server.MintDsDaemon $@ -------------------------------------------------------------------------------- /conf/mintds.yaml: -------------------------------------------------------------------------------- 1 | # MintDS config YAML 2 | 3 | server: 4 | # tcp connection port 5 | port: 7657 6 | # bind the server to an address 7 | bind-address: 0.0.0.0 8 | 9 | bloomfilter: 10 | elements: 16777216 # 2^24 11 | probability: 0.1 12 | 13 | countminsketch: 14 | width: 16777216 # 2^24 15 | depth: 4 -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/config/BloomFilterConfig.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.config; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | import lombok.ToString; 6 | 7 | @Getter 8 | @ToString 9 | @AllArgsConstructor 10 | public class BloomFilterConfig { 11 | 12 | private int elements; 13 | private double probability; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/config/ServerConfig.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.config; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | import lombok.ToString; 6 | 7 | @Getter 8 | @ToString 9 | @AllArgsConstructor 10 | public class ServerConfig { 11 | 12 | private String bindAddress = "0.0.0.0"; 13 | private int port = 7657; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /bin/mintds-cli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | HOST=localhost 4 | PORT=7657 5 | 6 | while [ $# -gt 0 ]; do 7 | COMMAND=$1 8 | case $COMMAND in 9 | --host) 10 | HOST=$2 11 | shift 2 12 | ;; 13 | --port) 14 | PORT=$2 15 | shift 2 16 | ;; 17 | *) 18 | break 19 | ;; 20 | esac 21 | done 22 | 23 | base_dir=$(dirname $0) 24 | 25 | exec $base_dir/mintds-run-class.sh com.arturmkrtchyan.mintds.cli.MintDsTerminal $HOST $PORT -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/MintDsServer.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server; 2 | 3 | import com.arturmkrtchyan.mintds.config.Configuration; 4 | import com.arturmkrtchyan.mintds.server.netty.NettyServer; 5 | 6 | public class MintDsServer { 7 | 8 | private final NettyServer server; 9 | 10 | public MintDsServer() { 11 | server = new NettyServer(); 12 | } 13 | 14 | public void start(final Configuration configuration) { 15 | server.start(configuration); 16 | } 17 | 18 | public void stop() { 19 | server.stop(); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/ResponseEncoder.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.response.Response; 4 | import io.netty.channel.ChannelHandler.Sharable; 5 | import io.netty.channel.ChannelHandlerContext; 6 | import io.netty.handler.codec.MessageToMessageEncoder; 7 | 8 | import java.util.List; 9 | 10 | @Sharable 11 | class ResponseEncoder extends MessageToMessageEncoder { 12 | 13 | @Override 14 | protected void encode(final ChannelHandlerContext ctx, final Response response, final List out) { 15 | out.add(response.toString().toLowerCase() + System.lineSeparator()); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/main/assembly/jar.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | all 6 | 7 | jar 8 | 9 | false 10 | 11 | 12 | / 13 | true 14 | true 15 | runtime 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/RequestDecoder.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.request.Request; 5 | import io.netty.channel.ChannelHandler.Sharable; 6 | import io.netty.channel.ChannelHandlerContext; 7 | import io.netty.handler.codec.MessageToMessageDecoder; 8 | 9 | import java.util.List; 10 | 11 | @Sharable 12 | class RequestDecoder extends MessageToMessageDecoder { 13 | 14 | @Override 15 | protected void decode(final ChannelHandlerContext channelHandlerContext, final String msg, final List out) { 16 | // parse the message to the right request. 17 | final Request request = DefaultRequest.fromString(msg); 18 | out.add(request); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /bin/mintds-run-class.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 1 ]; 4 | then 5 | echo "USAGE: $0 classname" 6 | exit 1 7 | fi 8 | 9 | base_dir=$(dirname $0)/.. 10 | 11 | MINTDS_JAR=$base_dir/target/mintds-0.1.1-SNAPSHOT-all.jar 12 | CLASSPATH=$MINTDS_JAR 13 | 14 | # Which java to use 15 | if [ -z "$JAVA_HOME" ]; then 16 | JAVA="java" 17 | else 18 | JAVA="$JAVA_HOME/bin/java" 19 | fi 20 | 21 | # Memory options 22 | if [ -z "$MINTDS_HEAP_OPTS" ]; then 23 | MINTDS_HEAP_OPTS="-Xms512m -Xmx2048m" 24 | fi 25 | 26 | # JVM performance options 27 | if [ -z "$MINTDS_JVM_PERFORMANCE_OPTS" ]; then 28 | MINTDS_JVM_PERFORMANCE_OPTS="-server -verbose:gc -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -Djava.awt.headless=true" 29 | fi 30 | 31 | #echo "$@" 32 | 33 | exec $JAVA $MINTDS_HEAP_OPTS $MINTDS_JVM_PERFORMANCE_OPTS -cp $CLASSPATH "$@" 34 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/NettyExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.response.FailureResponse; 4 | import io.netty.channel.ChannelHandler.Sharable; 5 | import io.netty.channel.ChannelHandlerAdapter; 6 | import io.netty.channel.ChannelHandlerContext; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | @Sharable 11 | class NettyExceptionHandler extends ChannelHandlerAdapter { 12 | 13 | private final static Logger logger = LoggerFactory.getLogger(NettyExceptionHandler.class); 14 | 15 | @Override 16 | public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) { 17 | logger.error("Exception is thrown in the pipeline.", cause); 18 | ctx.write(new FailureResponse(cause.getCause().getMessage())); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/MintDsDaemon.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server; 2 | 3 | import com.arturmkrtchyan.mintds.config.Configuration; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.io.IOException; 8 | 9 | public class MintDsDaemon { 10 | 11 | private final static Logger logger = LoggerFactory.getLogger(MintDsDaemon.class); 12 | 13 | public static void main(final String[] args) throws IOException { 14 | Configuration config = Configuration.valueOf(args[0]); 15 | 16 | System.out.println(config); 17 | 18 | final MintDsServer server = new MintDsServer(); 19 | logger.info("Starting MintDS Server."); 20 | server.start(config); 21 | 22 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 23 | logger.info("Stopping MintDS Server."); 24 | server.stop(); 25 | })); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/HyperLogLogStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.Request; 4 | import com.arturmkrtchyan.mintds.protocol.response.NumericResponse; 5 | import com.arturmkrtchyan.mintds.protocol.response.Response; 6 | import com.clearspring.analytics.stream.cardinality.HyperLogLog; 7 | 8 | class HyperLogLogStore extends AbstractKeyValueStore { 9 | 10 | public static final int DEFAULT_LOG2M = 16; 11 | 12 | public HyperLogLogStore() { 13 | } 14 | 15 | @Override 16 | public void add(final HyperLogLog log, final Request request) { 17 | log.offer(request.getValue().get()); 18 | } 19 | 20 | @Override 21 | public Response count(final HyperLogLog log, final Request request) { 22 | return new NumericResponse<>(log.cardinality()); 23 | } 24 | 25 | @Override 26 | protected HyperLogLog newElement() { 27 | return new HyperLogLog(DEFAULT_LOG2M); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/BloomFilterStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.config.BloomFilterConfig; 4 | import com.arturmkrtchyan.mintds.protocol.request.Request; 5 | import com.arturmkrtchyan.mintds.protocol.response.EnumResponse; 6 | import com.arturmkrtchyan.mintds.protocol.response.Response; 7 | import com.clearspring.analytics.stream.membership.BloomFilter; 8 | 9 | class BloomFilterStore extends AbstractKeyValueStore { 10 | 11 | private final BloomFilterConfig config; 12 | 13 | public BloomFilterStore(final BloomFilterConfig config) { 14 | this.config = config; 15 | } 16 | 17 | @Override 18 | public void add(final BloomFilter filter, final Request request) { 19 | filter.add(request.getValue().get()); 20 | } 21 | 22 | @Override 23 | public Response contains(final BloomFilter filter, final Request request) { 24 | return filter.isPresent(request.getValue().get()) ? EnumResponse.YES : EnumResponse.NO; 25 | } 26 | 27 | @Override 28 | protected BloomFilter newElement() { 29 | return new BloomFilter(config.getElements(), config.getProbability()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/CountMinSketchStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.Request; 4 | import com.arturmkrtchyan.mintds.protocol.response.NumericResponse; 5 | import com.arturmkrtchyan.mintds.protocol.response.Response; 6 | import com.clearspring.analytics.stream.frequency.CountMinSketch; 7 | 8 | import java.time.LocalTime; 9 | 10 | class CountMinSketchStore extends AbstractKeyValueStore { 11 | 12 | public static final int DEFAULT_WIDTH = 16_777_216; // 2^24 13 | public static final int DEFAULT_DEPTH = 4; 14 | 15 | public CountMinSketchStore() { 16 | } 17 | 18 | @Override 19 | public void add(final CountMinSketch sketch, final Request request) { 20 | sketch.add(request.getValue().get(), 1); 21 | } 22 | 23 | @Override 24 | public Response count(final CountMinSketch sketch, final Request request) { 25 | return new NumericResponse<>(sketch.estimateCount(request.getValue().get())); 26 | } 27 | 28 | @Override 29 | protected CountMinSketch newElement() { 30 | return new CountMinSketch(DEFAULT_DEPTH, DEFAULT_WIDTH, LocalTime.now().toSecondOfDay()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/NettyServerHandler.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.core.KeyValueStoreRouter; 4 | import com.arturmkrtchyan.mintds.protocol.request.Request; 5 | import com.arturmkrtchyan.mintds.protocol.response.Response; 6 | import io.netty.channel.ChannelHandler.Sharable; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.channel.SimpleChannelInboundHandler; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | @Sharable 13 | class NettyServerHandler extends SimpleChannelInboundHandler { 14 | 15 | private final static Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); 16 | 17 | private final KeyValueStoreRouter storeRouter; 18 | 19 | public NettyServerHandler(KeyValueStoreRouter storeRouter) { 20 | this.storeRouter = storeRouter; 21 | } 22 | 23 | @Override 24 | public void channelRead0(final ChannelHandlerContext ctx, final Request msg) { 25 | logger.debug("Received: " + msg.toString()); 26 | final Response response = storeRouter.route(msg); 27 | ctx.write(response); 28 | } 29 | 30 | @Override 31 | public void channelReadComplete(final ChannelHandlerContext ctx) { 32 | ctx.flush(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/KeyValueStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.Request; 4 | import com.arturmkrtchyan.mintds.protocol.response.EnumResponse; 5 | import com.arturmkrtchyan.mintds.protocol.response.Response; 6 | 7 | interface KeyValueStore { 8 | 9 | default public Response handle(final Request request) { 10 | switch (request.getCommand()) { 11 | case CREATE: 12 | return create(request); 13 | case EXISTS: 14 | return exists(request); 15 | case ADD: 16 | return add(request); 17 | case REMOVE: 18 | return remove(request); 19 | case CONTAINS: 20 | return contains(request); 21 | case COUNT: 22 | return count(request); 23 | case DROP: 24 | return drop(request); 25 | default: 26 | return EnumResponse.SUCCESS; 27 | } 28 | } 29 | 30 | Response create(Request request); 31 | 32 | Response exists(Request request); 33 | 34 | Response add(Request request); 35 | 36 | Response remove(Request request); 37 | 38 | Response contains(Request request); 39 | 40 | Response drop(Request request); 41 | 42 | Response count(Request request); 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/assembly/dist.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | tar.gz 7 | 8 | 9 | 10 | 11 | 12 | ${project.basedir} 13 | / 14 | 15 | README* 16 | LICENSE* 17 | NOTICE* 18 | 19 | 20 | 21 | ${project.basedir}/bin 22 | bin/ 23 | 24 | ** 25 | 26 | 27 | 28 | 29 | ${project.build.directory} 30 | / 31 | 32 | *-all.jar 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/KeyValueStoreRouter.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.config.Configuration; 4 | import com.arturmkrtchyan.mintds.protocol.request.DataStructure; 5 | import com.arturmkrtchyan.mintds.protocol.request.Request; 6 | import com.arturmkrtchyan.mintds.protocol.response.Response; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class KeyValueStoreRouter { 14 | 15 | private final static Logger logger = LoggerFactory.getLogger(KeyValueStoreRouter.class); 16 | 17 | private final Map storeRoutes; 18 | 19 | public KeyValueStoreRouter(final Configuration configuration) { 20 | storeRoutes = new HashMap<>(); 21 | storeRoutes.put(DataStructure.BloomFilter, 22 | new BloomFilterStore(configuration.getBloomFilterConfig())); 23 | storeRoutes.put(DataStructure.HyperLogLog, new HyperLogLogStore()); 24 | storeRoutes.put(DataStructure.CountMinSketch, new CountMinSketchStore()); 25 | storeRoutes.put(DataStructure.CountingBloomFilter, new CountingBloomFilterStore()); 26 | } 27 | 28 | public Response route(final Request request) { 29 | logger.debug("Handle request!"); 30 | return storeRoutes.get(request.getDataStructure()).handle(request); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/CountingBloomFilterStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.core.datastructure.CountingBloomFilter; 4 | import com.arturmkrtchyan.mintds.protocol.request.Request; 5 | import com.arturmkrtchyan.mintds.protocol.response.EnumResponse; 6 | import com.arturmkrtchyan.mintds.protocol.response.NumericResponse; 7 | import com.arturmkrtchyan.mintds.protocol.response.Response; 8 | 9 | class CountingBloomFilterStore extends AbstractKeyValueStore { 10 | 11 | public CountingBloomFilterStore() { 12 | } 13 | 14 | @Override 15 | public void add(final CountingBloomFilter filter, final Request request) { 16 | filter.add(request.getValue().get()); 17 | } 18 | 19 | @Override 20 | public Response remove(final CountingBloomFilter filter, final Request request) { 21 | filter.delete(request.getValue().get()); 22 | return EnumResponse.SUCCESS; 23 | } 24 | 25 | @Override 26 | public Response count(final CountingBloomFilter filter, final Request request) { 27 | return new NumericResponse<>(filter.count(request.getValue().get())); 28 | } 29 | 30 | @Override 31 | public Response contains(final CountingBloomFilter filter, final Request request) { 32 | return filter.isPresent(request.getValue().get()) ? EnumResponse.YES : EnumResponse.NO; 33 | } 34 | 35 | @Override 36 | protected CountingBloomFilter newElement() { 37 | return new CountingBloomFilter(1024, 1d); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/AbstractKeyValueStoreIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.client.MintDsClient; 4 | import com.arturmkrtchyan.mintds.config.Configuration; 5 | import com.arturmkrtchyan.mintds.server.MintDsServer; 6 | import org.junit.AfterClass; 7 | import org.junit.BeforeClass; 8 | 9 | import java.util.concurrent.CompletableFuture; 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.Executors; 12 | 13 | public class AbstractKeyValueStoreIT { 14 | 15 | private static MintDsServer server; 16 | protected static MintDsClient client; 17 | private static ExecutorService executor; 18 | 19 | public static final String DEFAULT_HOST = "127.0.0.1"; 20 | public static final int DEFAULT_PORT = 4444; 21 | 22 | @BeforeClass 23 | public static void beforeClass() throws Exception { 24 | startServer(); 25 | client = new MintDsClient.Builder() 26 | .host(DEFAULT_HOST) 27 | .port(DEFAULT_PORT) 28 | .numberOfThreads(1) 29 | .numberOfConnections(1) 30 | .build(); 31 | } 32 | 33 | @AfterClass 34 | public static void afterClass() throws Exception { 35 | client.close(); 36 | stopServer(); 37 | } 38 | 39 | protected static void startServer() throws Exception { 40 | System.setProperty("port", String.valueOf(DEFAULT_PORT)); 41 | executor = Executors.newFixedThreadPool(1); 42 | server = new MintDsServer(); 43 | CompletableFuture.runAsync(() -> 44 | server.start(Configuration.valueOf(DEFAULT_HOST, DEFAULT_PORT)), executor); 45 | Thread.sleep(3000); 46 | } 47 | 48 | protected static void stopServer() { 49 | server.stop(); 50 | executor.shutdown(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 9 | 10 | 11 | 12 | 13 | 14 | /var/log/mintds.log 15 | 16 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 17 | 18 | 19 | 21 | /srv/logs/mintds_%d{yyyy-MM-dd}.%i.log 22 | 23 | 24 | 10MB 25 | 26 | 27 | 30 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/NettyServer.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.config.Configuration; 4 | import com.arturmkrtchyan.mintds.config.ServerConfig; 5 | import com.arturmkrtchyan.mintds.core.KeyValueStoreRouter; 6 | import com.arturmkrtchyan.mintds.server.MintDsServer; 7 | import io.netty.bootstrap.ServerBootstrap; 8 | import io.netty.channel.EventLoopGroup; 9 | import io.netty.channel.nio.NioEventLoopGroup; 10 | import io.netty.channel.socket.nio.NioServerSocketChannel; 11 | import io.netty.handler.logging.LogLevel; 12 | import io.netty.handler.logging.LoggingHandler; 13 | import io.netty.util.concurrent.EventExecutorGroup; 14 | 15 | import java.util.Optional; 16 | 17 | public class NettyServer { 18 | 19 | private Optional bossGroup; 20 | private Optional workerGroup; 21 | 22 | public void start(final Configuration configuration) { 23 | bossGroup = Optional.of(new NioEventLoopGroup(1)); 24 | workerGroup = Optional.of(new NioEventLoopGroup()); 25 | try { 26 | final ServerBootstrap b = new ServerBootstrap(); 27 | b.group(bossGroup.get(), workerGroup.get()) 28 | .channel(NioServerSocketChannel.class) 29 | .handler(new LoggingHandler(LogLevel.INFO)) 30 | .childHandler(new NettyServerInitializer(new KeyValueStoreRouter(configuration))); 31 | 32 | final ServerConfig serverConfig = configuration.getServerConfig(); 33 | b.bind(serverConfig.getBindAddress(), serverConfig.getPort()) 34 | .sync().channel().closeFuture().sync(); 35 | } catch (InterruptedException e) { 36 | throw new RuntimeException(e); 37 | } finally { 38 | stop(); 39 | } 40 | } 41 | 42 | public void stop() { 43 | bossGroup.ifPresent(EventExecutorGroup::shutdownGracefully); 44 | workerGroup.ifPresent(EventExecutorGroup::shutdownGracefully); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/HyperLogLogIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.response.Response; 5 | import org.javatuples.Pair; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.concurrent.CompletableFuture; 12 | 13 | public class HyperLogLogIT extends AbstractKeyValueStoreIT { 14 | 15 | private List> happyUseCaseData() { 16 | return Arrays.asList( 17 | new Pair<>("create hyperloglog mylog", Response.fromString("success")), 18 | new Pair<>("create hyperloglog mylog", Response.fromString("exists")), 19 | new Pair<>("exists hyperloglog mylog", Response.fromString("yes")), 20 | new Pair<>("exists hyperloglog newlog", Response.fromString("no")), 21 | new Pair<>("add hyperloglog mylog myvalue", Response.fromString("success")), 22 | new Pair<>("count hyperloglog mylog", Response.fromString("1")), 23 | new Pair<>("drop hyperloglog mylog", Response.fromString("success")), 24 | new Pair<>("drop hyperloglog newlog", Response.fromString("non_existent")), 25 | new Pair<>("count hyperloglog nolog", Response.fromString("failure nolog doesn't exist.")) 26 | ); 27 | } 28 | 29 | @Test 30 | public void happyUseCase() throws Exception { 31 | List> data = happyUseCaseData(); 32 | data.stream().forEach(pair -> { 33 | CompletableFuture future = client.send(DefaultRequest.fromString(pair.getValue0())); 34 | try { 35 | Assert.assertEquals("Sending request->" + pair.getValue0(), 36 | pair.getValue1(), future.get()); 37 | } catch (Exception e) { 38 | e.printStackTrace(); 39 | } 40 | }); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/cli/MintDsTerminal.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.cli; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.response.Response; 5 | import jline.TerminalFactory; 6 | import jline.console.ConsoleReader; 7 | 8 | import java.util.Optional; 9 | import java.util.concurrent.CompletableFuture; 10 | 11 | import com.arturmkrtchyan.mintds.client.MintDsClient; 12 | 13 | public class MintDsTerminal { 14 | 15 | private static final String DEFAULT_HOST = "localhost"; 16 | private static final int DEFAULT_PORT = 7657; 17 | 18 | public static void main(String[] args) throws Exception { 19 | 20 | final Optional host = args.length > 0 ? Optional.of(args[0]) : Optional.empty(); 21 | final Optional port = args.length > 1 ? Optional.of(Integer.valueOf(args[1])) : Optional.empty(); 22 | 23 | try { 24 | ConsoleReader console = new ConsoleReader(null, System.in, System.out, null); 25 | console.setPrompt("\nmintDS> "); 26 | console.setBellEnabled(false); 27 | 28 | MintDsClient client = new MintDsClient.Builder() 29 | .host(host.orElse(DEFAULT_HOST)) 30 | .port(port.orElse(DEFAULT_PORT)) 31 | .numberOfThreads(1) 32 | .numberOfConnections(1) 33 | .build(); 34 | 35 | for (; ;) { 36 | String line = console.readLine(); 37 | if (line == null || "bye".equals(line.toLowerCase())) { 38 | break; 39 | } 40 | 41 | if (line.trim().isEmpty()) { 42 | continue; 43 | } 44 | 45 | // Waits for the response 46 | CompletableFuture future = client.send(DefaultRequest.fromString(line)); 47 | System.out.println(future.get()); 48 | } 49 | 50 | client.close(); 51 | 52 | } finally { 53 | TerminalFactory.get().restore(); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/BloomFilterIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.response.Response; 5 | import org.javatuples.Pair; 6 | import org.junit.*; 7 | 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.concurrent.CompletableFuture; 11 | 12 | public class BloomFilterIT extends AbstractKeyValueStoreIT { 13 | 14 | private List> happyUseCaseData() { 15 | return Arrays.asList( 16 | new Pair<>("create bloomfilter myfilter", Response.fromString("success")), 17 | new Pair<>("create bloomfilter myfilter", Response.fromString("exists")), 18 | new Pair<>("exists bloomfilter myfilter", Response.fromString("yes")), 19 | new Pair<>("exists bloomfilter newfilter", Response.fromString("no")), 20 | new Pair<>("add bloomfilter myfilter myvalue", Response.fromString("success")), 21 | new Pair<>("contains bloomfilter myfilter myvalue", Response.fromString("yes")), 22 | new Pair<>("contains bloomfilter myfilter mynewvalue", Response.fromString("no")), 23 | new Pair<>("drop bloomfilter myfilter", Response.fromString("success")), 24 | new Pair<>("drop bloomfilter mynewfilter", Response.fromString("non_existent")), 25 | new Pair<>("contains bloomfilter nofilter myvalue", 26 | Response.fromString("failure nofilter doesn't exist.")) 27 | ); 28 | } 29 | 30 | @Test 31 | public void happyUseCase() throws Exception { 32 | List> data = happyUseCaseData(); 33 | data.stream().forEach(pair -> { 34 | CompletableFuture future = client.send(DefaultRequest.fromString(pair.getValue0())); 35 | try { 36 | Assert.assertEquals("Sending request->" + pair.getValue0(), 37 | pair.getValue1(), future.get()); 38 | } catch (Exception e) { 39 | e.printStackTrace(); 40 | } 41 | }); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/CountMinSktechIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.response.Response; 5 | import org.javatuples.Pair; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.concurrent.CompletableFuture; 12 | 13 | public class CountMinSktechIT extends AbstractKeyValueStoreIT { 14 | 15 | private List> happyUseCaseData() { 16 | return Arrays.asList( 17 | new Pair<>("create countminsketch mysketch", Response.fromString("success")), 18 | new Pair<>("create countminsketch mysketch", Response.fromString("exists")), 19 | new Pair<>("exists countminsketch mysketch", Response.fromString("yes")), 20 | new Pair<>("exists countminsketch newsketch", Response.fromString("no")), 21 | new Pair<>("add countminsketch mysketch myvalue", Response.fromString("success")), 22 | new Pair<>("add countminsketch mysketch myvalue", Response.fromString("success")), 23 | new Pair<>("add countminsketch mysketch myvalue", Response.fromString("success")), 24 | new Pair<>("count countminsketch mysketch myvalue", Response.fromString("3")), 25 | new Pair<>("count countminsketch mysketch mynewvalue", Response.fromString("0")), 26 | new Pair<>("drop countminsketch mysketch", Response.fromString("success")), 27 | new Pair<>("drop countminsketch mysketch", Response.fromString("non_existent")), 28 | new Pair<>("count countminsketch nosketch myvalue", 29 | Response.fromString("failure nosketch doesn't exist.")) 30 | ); 31 | } 32 | 33 | @Test 34 | public void happyUseCase() throws Exception { 35 | List> data = happyUseCaseData(); 36 | data.stream().forEach(pair -> { 37 | CompletableFuture future = client.send(DefaultRequest.fromString(pair.getValue0())); 38 | try { 39 | Assert.assertEquals("Sending request->" + pair.getValue0(), 40 | pair.getValue1(), future.get()); 41 | } catch (Exception e) { 42 | e.printStackTrace(); 43 | } 44 | }); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/server/netty/NettyServerInitializer.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.server.netty; 2 | 3 | import com.arturmkrtchyan.mintds.core.KeyValueStoreRouter; 4 | import io.netty.channel.ChannelInitializer; 5 | import io.netty.channel.ChannelPipeline; 6 | import io.netty.channel.socket.SocketChannel; 7 | import io.netty.handler.codec.DelimiterBasedFrameDecoder; 8 | import io.netty.handler.codec.Delimiters; 9 | import io.netty.handler.codec.string.StringDecoder; 10 | import io.netty.handler.codec.string.StringEncoder; 11 | import io.netty.util.CharsetUtil; 12 | import io.netty.util.concurrent.DefaultEventExecutorGroup; 13 | import io.netty.util.concurrent.EventExecutorGroup; 14 | 15 | class NettyServerInitializer extends ChannelInitializer { 16 | 17 | // Use internal executor if this doesn't scale as netty pins connections to threads. 18 | private final EventExecutorGroup group = new DefaultEventExecutorGroup(16); 19 | 20 | private final StringDecoder stringDecoder; 21 | private final StringEncoder stringEncoder; 22 | private final RequestDecoder requestDecoder; 23 | private final ResponseEncoder responseEncoder; 24 | private final NettyServerHandler serverHandler; 25 | private final NettyExceptionHandler exceptionHandler; 26 | 27 | public NettyServerInitializer(KeyValueStoreRouter storeRouter) { 28 | stringDecoder = new StringDecoder(CharsetUtil.UTF_8); 29 | stringEncoder = new StringEncoder(CharsetUtil.UTF_8); 30 | requestDecoder = new RequestDecoder(); 31 | responseEncoder = new ResponseEncoder(); 32 | serverHandler = new NettyServerHandler(storeRouter); 33 | exceptionHandler = new NettyExceptionHandler(); 34 | } 35 | 36 | @Override 37 | protected void initChannel(final SocketChannel ch) throws Exception { 38 | 39 | final ChannelPipeline pipeline = ch.pipeline(); 40 | 41 | // decoders 42 | // Add the text line codec combination first, 43 | pipeline.addLast("framer", new DelimiterBasedFrameDecoder(8192, Delimiters.lineDelimiter())); 44 | pipeline.addLast("stringDecoder", stringDecoder); 45 | pipeline.addLast("requestDecoder", requestDecoder); 46 | 47 | // encoders 48 | pipeline.addLast("stringEncoder", stringEncoder); 49 | pipeline.addLast("responseEncoder", responseEncoder); 50 | 51 | 52 | // business logic handler 53 | pipeline.addLast(group, "serverHandler", serverHandler); 54 | pipeline.addLast("exceptionHandler", exceptionHandler); 55 | 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/MultithreadedAccessIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.client.MintDsClient; 4 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 5 | import com.arturmkrtchyan.mintds.protocol.response.Response; 6 | import org.javatuples.Pair; 7 | import org.junit.AfterClass; 8 | import org.junit.Assert; 9 | import org.junit.BeforeClass; 10 | import org.junit.Test; 11 | 12 | import java.util.Arrays; 13 | import java.util.List; 14 | import java.util.concurrent.CompletableFuture; 15 | 16 | public class MultithreadedAccessIT extends AbstractKeyValueStoreIT { 17 | 18 | private static MintDsClient multithreadedClient; 19 | 20 | @BeforeClass 21 | public static void beforeClass() throws Exception { 22 | startServer(); 23 | multithreadedClient = new MintDsClient.Builder() 24 | .host(DEFAULT_HOST) 25 | .port(DEFAULT_PORT) 26 | .numberOfThreads(4) 27 | .numberOfConnections(32) 28 | .build(); 29 | } 30 | 31 | @AfterClass 32 | public static void afterClass() throws Exception { 33 | multithreadedClient.close(); 34 | stopServer(); 35 | } 36 | 37 | private List> happyUseCaseData() { 38 | return Arrays.asList( 39 | new Pair<>("create bloomfilter myfilter", Response.fromString("success")), 40 | new Pair<>("create bloomfilter myfilter", Response.fromString("exists")), 41 | new Pair<>("exists bloomfilter myfilter", Response.fromString("yes")), 42 | new Pair<>("exists bloomfilter newfilter", Response.fromString("no")), 43 | new Pair<>("add bloomfilter myfilter myvalue", Response.fromString("success")), 44 | new Pair<>("contains bloomfilter myfilter myvalue", Response.fromString("yes")), 45 | new Pair<>("contains bloomfilter myfilter mynewvalue", Response.fromString("no")), 46 | new Pair<>("drop bloomfilter myfilter", Response.fromString("success")), 47 | new Pair<>("drop bloomfilter mynewfilter", Response.fromString("non_existent")) 48 | ); 49 | } 50 | 51 | @Test 52 | public void happyUseCase() throws Exception { 53 | List> data = happyUseCaseData(); 54 | data.stream().forEach(pair -> { 55 | CompletableFuture future = multithreadedClient.send(DefaultRequest.fromString(pair.getValue0())); 56 | try { 57 | Assert.assertEquals("Sending request->" + pair.getValue0(), 58 | pair.getValue1(), future.get()); 59 | } catch (Exception e) { 60 | e.printStackTrace(); 61 | } 62 | }); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/com/arturmkrtchyan/mintds/integration/CountingBloomFilterIT.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.integration; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.DefaultRequest; 4 | import com.arturmkrtchyan.mintds.protocol.response.Response; 5 | import org.javatuples.Pair; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.concurrent.CompletableFuture; 12 | 13 | public class CountingBloomFilterIT extends AbstractKeyValueStoreIT { 14 | 15 | private List> happyUseCaseData() { 16 | return Arrays.asList( 17 | new Pair<>("create countingbloomfilter myfilter", Response.fromString("success")), 18 | new Pair<>("create countingbloomfilter myfilter", Response.fromString("exists")), 19 | new Pair<>("exists countingbloomfilter myfilter", Response.fromString("yes")), 20 | new Pair<>("exists countingbloomfilter newfilter", Response.fromString("no")), 21 | new Pair<>("add countingbloomfilter myfilter myvalue", Response.fromString("success")), 22 | new Pair<>("contains countingbloomfilter myfilter myvalue", Response.fromString("yes")), 23 | new Pair<>("contains countingbloomfilter myfilter mynewvalue", Response.fromString("no")), 24 | new Pair<>("count countingbloomfilter myfilter myvalue", Response.fromString("1")), 25 | new Pair<>("add countingbloomfilter myfilter myvalue", Response.fromString("success")), 26 | new Pair<>("count countingbloomfilter myfilter myvalue", Response.fromString("2")), 27 | new Pair<>("remove countingbloomfilter myfilter myvalue", Response.fromString("success")), 28 | new Pair<>("count countingbloomfilter myfilter myvalue", Response.fromString("1")), 29 | new Pair<>("drop countingbloomfilter myfilter", Response.fromString("success")), 30 | new Pair<>("drop countingbloomfilter mynewfilter", Response.fromString("non_existent")), 31 | new Pair<>("contains countingbloomfilter nofilter myvalue", 32 | Response.fromString("failure nofilter doesn't exist.")) 33 | ); 34 | } 35 | 36 | @Test 37 | public void happyUseCase() throws Exception { 38 | List> data = happyUseCaseData(); 39 | data.stream().forEach(pair -> { 40 | CompletableFuture future = client.send(DefaultRequest.fromString(pair.getValue0())); 41 | try { 42 | Assert.assertEquals("Sending request->" + pair.getValue0(), 43 | pair.getValue1(), future.get()); 44 | } catch (Exception e) { 45 | e.printStackTrace(); 46 | } 47 | }); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/config/Configuration.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.config; 2 | 3 | import lombok.Getter; 4 | import lombok.ToString; 5 | import org.yaml.snakeyaml.Yaml; 6 | import org.yaml.snakeyaml.representer.Representer; 7 | 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.nio.file.Files; 11 | import java.nio.file.Paths; 12 | import java.util.Map; 13 | 14 | @Getter 15 | @ToString 16 | public class Configuration { 17 | 18 | public static final int DEFAULT_NUMBER_OF_ELEMENTS = 16_777_216; // 2^24 19 | public static final double DEFAULT_FALSE_POSITIVE_PROBABILITY = 0.1; 20 | 21 | private ServerConfig serverConfig; 22 | private BloomFilterConfig bloomFilterConfig; 23 | 24 | @SuppressWarnings("unchecked") 25 | private Configuration(final String path) throws IOException { 26 | final Map configMap = loadYaml(path); 27 | 28 | serverConfig = readServerConfig(configMap); 29 | bloomFilterConfig = readBloomFilterConfig(configMap); 30 | } 31 | 32 | private Configuration() { 33 | this("0.0.0.0", 7657); 34 | } 35 | 36 | private Configuration(final String bindAddress, final int port) { 37 | serverConfig = new ServerConfig(bindAddress, port); 38 | bloomFilterConfig = new BloomFilterConfig(DEFAULT_NUMBER_OF_ELEMENTS, DEFAULT_FALSE_POSITIVE_PROBABILITY); 39 | } 40 | 41 | public static Configuration valueOf(final String path) throws IOException { 42 | return new Configuration(path); 43 | } 44 | 45 | public static Configuration valueOf(final String bindAddress, final int port) { 46 | return new Configuration(bindAddress, port); 47 | } 48 | 49 | @SuppressWarnings("unchecked") 50 | private Map loadYaml(final String path) throws IOException { 51 | // yaml parsing 52 | Representer representer = new Representer(); 53 | representer.getPropertyUtils().setSkipMissingProperties(true); 54 | Yaml yaml = new Yaml(representer); 55 | try( InputStream in = Files.newInputStream(Paths.get(path)) ) { 56 | return (Map) yaml.load(in); 57 | } 58 | } 59 | 60 | @SuppressWarnings("unchecked") 61 | private ServerConfig readServerConfig(Map configMap) { 62 | final Map serverConfigMap = (Map) configMap.get("server"); 63 | final int port = (Integer)serverConfigMap.get("port"); 64 | final String bindAddress = (String)serverConfigMap.get("bind-address"); 65 | return new ServerConfig(bindAddress, port); 66 | } 67 | 68 | @SuppressWarnings("unchecked") 69 | private BloomFilterConfig readBloomFilterConfig(Map configMap) { 70 | final Map bloomFilterConfigMap = (Map) configMap.get("bloomfilter"); 71 | final int elements = (Integer) bloomFilterConfigMap.get("elements"); 72 | final double probability = (Double)bloomFilterConfigMap.get("probability"); 73 | return new BloomFilterConfig(elements, probability); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/AbstractKeyValueStore.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core; 2 | 3 | import com.arturmkrtchyan.mintds.protocol.request.Request; 4 | import com.arturmkrtchyan.mintds.protocol.response.EnumResponse; 5 | import com.arturmkrtchyan.mintds.protocol.response.FailureResponse; 6 | import com.arturmkrtchyan.mintds.protocol.response.Response; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | abstract class AbstractKeyValueStore implements KeyValueStore { 12 | 13 | protected final Map map = new ConcurrentHashMap<>(); 14 | 15 | @Override 16 | public Response create(final Request request) { 17 | if (map.containsKey(request.getKey())) { 18 | return EnumResponse.EXISTS; 19 | } 20 | map.put(request.getKey(), newElement()); 21 | return EnumResponse.SUCCESS; 22 | } 23 | 24 | @Override 25 | public Response add(final Request request) { 26 | final E element = map.get(request.getKey()); 27 | if (element != null) { 28 | add(element, request); 29 | return EnumResponse.SUCCESS; 30 | } 31 | return new FailureResponse("Failure " + request.getKey() + " doesn't exist."); 32 | } 33 | 34 | @Override 35 | public Response remove(final Request request) { 36 | final E element = map.get(request.getKey()); 37 | if (element != null) { 38 | return remove(element, request); 39 | } 40 | return new FailureResponse("Failure " + request.getKey() + " doesn't exist."); 41 | } 42 | 43 | @Override 44 | public Response exists(final Request request) { 45 | return map.containsKey(request.getKey()) ? EnumResponse.YES : EnumResponse.NO; 46 | } 47 | 48 | @Override 49 | public Response drop(final Request request) { 50 | if (!map.containsKey(request.getKey())) { 51 | return EnumResponse.NON_EXISTENT; 52 | } 53 | map.remove(request.getKey()); 54 | return EnumResponse.SUCCESS; 55 | } 56 | 57 | @Override 58 | public Response count(final Request request) { 59 | final E element = map.get(request.getKey()); 60 | if (element != null) { 61 | return count(element, request); 62 | } 63 | return new FailureResponse("Failure " + request.getKey() + " doesn't exist."); 64 | } 65 | 66 | @Override 67 | public Response contains(final Request request) { 68 | final E element = map.get(request.getKey()); 69 | if (element != null) { 70 | return contains(element, request); 71 | } 72 | return new FailureResponse("Failure " + request.getKey() + " doesn't exist."); 73 | } 74 | 75 | 76 | protected Response count(final E element, final Request request) { 77 | return new FailureResponse("Failure unsupported command"); 78 | } 79 | 80 | protected Response remove(final E element, final Request request) { 81 | return new FailureResponse("Failure unsupported command"); 82 | } 83 | 84 | protected Response contains(final E element, final Request request) { 85 | return new FailureResponse("Failure unsupported command"); 86 | } 87 | 88 | protected abstract E newElement(); 89 | 90 | protected abstract void add(E element, Request request); 91 | 92 | } 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/mintDS/mintds.svg)](https://travis-ci.org/mintDS/mintds) 2 | 3 | What is mintDS? 4 | -------------- 5 | 6 | mintDS is a *probabilistic data structures* server. 7 | 8 | What are the mintDS data structures? 9 | -------------- 10 | ➤ **Bloom Filter** is a space-efficient probabilistic data structure which is used to test whether an element is a member of a set. Membership query returns either "possibly in set" or "definitely not in set". The probability of false positives can be easily configured. 11 | 12 | ➤ **Counting Bloom Filter** Bloom Filter which supports deletions and counting. 13 | 14 | ➤ **HyperLogLog** is a space-efficient probabilistic data structure which is used to get the approximate number of distinct elements in a multiset. The accuracy can be easily configured. 15 | 16 | ➤ **Count–min sketch** is a space-efficient probabilistic data structure which is used to get the approximate frequencies of specific elements in a multiset. The accuracy can be easily configured. 17 | 18 | Performance 19 | -------------- 20 | mintDS is super fast. Here are some numbers based on aws **c4.xlarge** instance: 21 | - **16ms** to asynchronously send **100K** messages. 22 | - **1s** to asynchronously send and receive **70K** messages. 23 | 24 | **Want to try it out ?** 25 | 26 | Check out mintd-java and run -> `java -Dconnections=100 -Dthreads=4 -Dhost= -cp target/mintds-java-0.1.2-SNAPSHOT-all.jar com.arturmkrtchyan.mintds.benchmark.BloomFilterBenchmark` 27 | 28 | Playing with mintDS 29 | -------------- 30 | To run mintDS server simply type: 31 | ```shell 32 | ./bin/mintds-start.sh conf/mintds.yaml 33 | ``` 34 | 35 | After starting the server you can use mintds-cli to play with mintDS. 36 | ```shell 37 | ./bin/mintds-cli.sh --host localhost --port 7657 38 | ``` 39 | 40 | ####Bloom Filter 41 | 42 | ```shell 43 | 44 | mintDS> create bloomfilter myfilter 45 | SUCCESS 46 | 47 | mintDS> exists bloomfilter myfilter 48 | YES 49 | 50 | mintDS> add bloomfilter myfilter myvalue 51 | SUCCESS 52 | 53 | mintDS> contains bloomfilter myfilter myvalue 54 | YES 55 | 56 | mintDS> contains bloomfilter myfilter mynewvalue 57 | NO 58 | 59 | mintDS> drop bloomfilter myfilter 60 | SUCCESS 61 | 62 | mintDS> 63 | ``` 64 | 65 | ####Counting Bloom Filter 66 | 67 | ```shell 68 | 69 | mintDS> create countingbloomfilter myfilter 70 | SUCCESS 71 | 72 | mintDS> exists countingbloomfilter myfilter 73 | YES 74 | 75 | mintDS> add countingbloomfilter myfilter myvalue 76 | SUCCESS 77 | 78 | mintDS> contains countingbloomfilter myfilter myvalue 79 | YES 80 | 81 | mintDS> contains countingbloomfilter myfilter mynewvalue 82 | NO 83 | 84 | mintDS> count countingbloomfilter myfilter myvalue 85 | 1 86 | 87 | mintDS> add countingbloomfilter myfilter myvalue 88 | SUCCESS 89 | 90 | mintDS> count countingbloomfilter myfilter myvalue 91 | 2 92 | 93 | mintDS> remove countingbloomfilter myfilter myvalue 94 | SUCCESS 95 | 96 | mintDS> count countingbloomfilter myfilter myvalue 97 | 1 98 | 99 | mintDS> drop countingbloomfilter myfilter 100 | SUCCESS 101 | 102 | mintDS> 103 | ``` 104 | 105 | ####HyperLogLog 106 | 107 | ```shell 108 | mintDS> create hyperloglog mylog 109 | SUCCESS 110 | 111 | mintDS> exists hyperloglog mylog 112 | YES 113 | 114 | mintDS> add hyperloglog mylog myvalue 115 | SUCCESS 116 | 117 | mintDS> add hyperloglog mylog mynewvalue 118 | SUCCESS 119 | 120 | mintDS> count hyperloglog mylog 121 | 2 122 | 123 | mintDS> drop hyperloglog mylog 124 | SUCCESS 125 | 126 | mintDS> 127 | ``` 128 | 129 | ####Count-Min Sketch 130 | 131 | ```shell 132 | mintDS> create countminsketch mysketch 133 | SUCCESS 134 | 135 | mintDS> exists countminsketch mysketch 136 | YES 137 | 138 | mintDS> add countminsketch mysketch myvalue 139 | SUCCESS 140 | 141 | mintDS> count countminsketch mysketch myvalue 142 | 1 143 | 144 | mintDS> add countminsketch mysketch myvalue 145 | SUCCESS 146 | 147 | mintDS> count countminsketch mysketch myvalue 148 | 2 149 | 150 | mintDS> drop countminsketch mysketch 151 | SUCCESS 152 | 153 | mintDS> 154 | ``` 155 | 156 | Client Implementations 157 | -------------- 158 | | Client | Description | 159 | |:------------|:--------------------------------------------:| 160 | | [mintds-java](https://github.com/mintDS/mintds-java) | Asynchronous Java client library for mintDS. | 161 | 162 | Check out the protocol description [here](https://github.com/mintDS/mintds-protocol). 163 | 164 | Credits 165 | -------------- 166 | Datastructures are based on [addthis/stream-lib](https://github.com/addthis/stream-lib). 167 | 168 | References 169 | -------------- 170 | The list of related open-source projects and scientific papers which mintDS makes use of: 171 | - **Bloom Filter** by Burton H. Bloom - [Space/Time Trade-offs in Hash Coding with Allowable Errors] (https://www.cs.upc.edu/~diaz/p422-bloom.pdf) 172 | - **Counting Bloom Filter** by Flavio Bonomi, et al - [An Improved Construction for Counting Bloom Filters] (https://www.eecs.berkeley.edu/~sylvia/cs268-2013/papers/countingbloom.pdf) 173 | - **HyperLogLog** by Philippe Flajolet, et al - [HyperLogLog: the analysis of a near-optimal 174 | cardinality estimation algorithm] (http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) 175 | - **Count-Min Sketch** by by Graham Cormode, et al - [Count-Min Sketch] (http://dimacs.rutgers.edu/~graham/pubs/papers/cmencyc.pdf) 176 | -------------------------------------------------------------------------------- /src/main/java/com/arturmkrtchyan/mintds/core/datastructure/CountingBloomFilter.java: -------------------------------------------------------------------------------- 1 | package com.arturmkrtchyan.mintds.core.datastructure; 2 | 3 | 4 | import com.clearspring.analytics.hash.MurmurHash; 5 | 6 | import java.io.UnsupportedEncodingException; 7 | 8 | public class CountingBloomFilter { 9 | 10 | // Storage 11 | private long[] buckets; 12 | 13 | // 4bit buckets, bucket can count to 15 14 | private final static long BUCKET_MAX_COUNT = 15; 15 | 16 | private final int numElements; 17 | 18 | private int hashCount = 2; // TODO calculate 19 | 20 | public CountingBloomFilter(final int numElements, final double maxFalsePosProbability) { 21 | this.numElements = numElements; 22 | buckets = new long[calculateBucketSize(numElements)]; 23 | System.out.println(buckets.length); 24 | } 25 | 26 | // how many longs (64 bits) would take to hold numElements buckets (4 bits) 27 | private int calculateBucketSize(final int numElements) { 28 | return ((numElements - 1) >>> 4) + 1; 29 | } 30 | 31 | public boolean isPresent(String key) { 32 | int[] hashes = getHashBuckets(key, hashCount, numElements); 33 | for(int hash : hashes) { 34 | // find the bucket 35 | int bucketIndex = hash >> 4; // div 16 ; to find the index inside buckets 36 | System.out.println("hash: " + hash); 37 | System.out.println("index: " + bucketIndex); 38 | int bucketShift = (hash & 0x0f) << 2; // (mod 16) * 4 ; index inside a single long 39 | System.out.println("shift: " + bucketShift); 40 | 41 | long bucketMask = 15L << bucketShift; 42 | System.out.println("mask: " + bucketMask); 43 | System.out.println("exists: " + (buckets[bucketIndex] & bucketMask)); 44 | 45 | if((buckets[bucketIndex] & bucketMask) == 0) { 46 | return false; 47 | } 48 | } 49 | return true; 50 | } 51 | 52 | public void add(String key) { 53 | int[] hashes = getHashBuckets(key, hashCount, numElements); 54 | for(int hash : hashes) { 55 | // find the bucket 56 | int bucketIndex = hash >> 4; // div 16 ; to find the index inside buckets 57 | int bucketShift = (hash & 0x0f) << 2; // (mod 16) * 4 ; index inside a single long 58 | 59 | long bucketMask = 15L << bucketShift; 60 | long bucketValue = (buckets[bucketIndex] & bucketMask) >>> bucketShift; 61 | 62 | // only increment if the count in the bucket is less than BUCKET_MAX_COUNT 63 | if(bucketValue < BUCKET_MAX_COUNT) { 64 | // increment by 1 65 | buckets[bucketIndex] = (buckets[bucketIndex] & ~bucketMask) | ((bucketValue + 1) << bucketShift); 66 | } 67 | } 68 | } 69 | 70 | public void delete(String key) { 71 | int[] hashes = getHashBuckets(key, hashCount, numElements); 72 | for(int hash : hashes) { 73 | // find the bucket 74 | int bucketIndex = hash >> 4; // div 16 ; to find the index inside buckets 75 | int bucketShift = (hash & 0x0f) << 2; // (mod 16) * 4 ; index inside a single long 76 | 77 | long bucketMask = 15L << bucketShift; 78 | long bucketValue = (buckets[bucketIndex] & bucketMask) >>> bucketShift; 79 | 80 | // only decrement if the count in the bucket is between 0 and BUCKET_MAX_COUNT 81 | if(bucketValue >= 1 && bucketValue < BUCKET_MAX_COUNT) { 82 | // increment by 1 83 | buckets[bucketIndex] = (buckets[bucketIndex] & ~bucketMask) | ((bucketValue - 1) << bucketShift); 84 | } 85 | } 86 | } 87 | 88 | public int count(String key) { 89 | int res = Integer.MAX_VALUE; 90 | int[] hashes = getHashBuckets(key, hashCount, numElements); 91 | for(int hash : hashes) { 92 | // find the bucket 93 | int bucketIndex = hash >> 4; // div 16 ; to find the index inside buckets 94 | int bucketShift = (hash & 0x0f) << 2; // (mod 16) * 4 ; index inside a single long 95 | 96 | long bucketMask = 15L << bucketShift; 97 | long bucketValue = (buckets[bucketIndex] & bucketMask) >>> bucketShift; 98 | 99 | if (bucketValue < res) res = (int)bucketValue; 100 | } 101 | if (res != Integer.MAX_VALUE) { 102 | return res; 103 | } else { 104 | return 0; 105 | } 106 | } 107 | 108 | public int[] getHashBuckets(String key, int hashCount, int max) { 109 | byte[] b; 110 | try { 111 | b = key.getBytes("UTF-8"); 112 | } catch (UnsupportedEncodingException e) { 113 | throw new RuntimeException(e); 114 | } 115 | return getHashBuckets(b, hashCount, max); 116 | } 117 | 118 | int[] getHashBuckets(byte[] b, int hashCount, int max) { 119 | int[] result = new int[hashCount]; 120 | int hash1 = MurmurHash.hash(b, b.length, 0); 121 | int hash2 = MurmurHash.hash(b, b.length, hash1); 122 | for (int i = 0; i < hashCount; i++) { 123 | result[i] = Math.abs((hash1 + i * hash2) % max); 124 | } 125 | return result; 126 | } 127 | 128 | public static void main(String[] args) { 129 | CountingBloomFilter filter = new CountingBloomFilter(12, 1d); 130 | filter.add("test"); 131 | System.out.println(filter.isPresent("test")); 132 | System.out.println(filter.count("test")); 133 | filter.add("test"); 134 | filter.add("test"); 135 | filter.add("test"); 136 | filter.add("test"); 137 | System.out.println(filter.count("test")); 138 | filter.delete("test"); 139 | System.out.println(filter.count("test")); 140 | 141 | System.out.println(filter.isPresent("test1")); 142 | } 143 | 144 | } 145 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.arturmkrtchyan.mintds 5 | mintds 6 | jar 7 | 0.1.1-SNAPSHOT 8 | 9 | mintds 10 | 11 | Probabilistic data structures server. 12 | The data model is key-value, where values are: Bloomfilters, LinearCounters, HyperLogLogs, CountMinSketches and 13 | StreamSummaries. 14 | 15 | http://maven.apache.org 16 | 17 | 18 | 19 | Apache License 2.0 20 | http://www.apache.org/licenses/LICENSE-2.0 21 | repo 22 | 23 | 24 | 25 | 26 | 27 | Artur Mkrtchyan 28 | 29 | 30 | 31 | 32 | scm:git:git@github.com:mintDS/mintds.git 33 | scm:git:git@github.com:mintDS/mintds.git 34 | git@github.com:mintDS/mintds.git 35 | HEAD 36 | 37 | 38 | 39 | UTF-8 40 | UTF-8 41 | 1.8 42 | true 43 | true 44 | true 45 | false 46 | 47 | 48 | 49 | 50 | 51 | com.clearspring.analytics 52 | stream 53 | 2.9.0 54 | 55 | 56 | 57 | jline 58 | jline 59 | 2.13 60 | 61 | 62 | 63 | 64 | io.netty 65 | netty-all 66 | 4.0.32.Final 67 | 68 | 69 | 70 | com.arturmkrtchyan.mintds 71 | mintds-protocol 72 | 0.1.6 73 | 74 | 75 | 76 | com.arturmkrtchyan.mintds 77 | mintds-java 78 | 0.1.1 79 | 80 | 81 | 82 | net.openhft 83 | zero-allocation-hashing 84 | 0.4 85 | 86 | 87 | 88 | 89 | org.slf4j 90 | slf4j-api 91 | 1.7.12 92 | 93 | 94 | 95 | ch.qos.logback 96 | logback-classic 97 | 1.1.3 98 | 99 | 100 | 101 | ch.qos.logback 102 | logback-core 103 | 1.1.3 104 | 105 | 106 | 107 | org.yaml 108 | snakeyaml 109 | 1.16 110 | 111 | 112 | 113 | org.projectlombok 114 | lombok 115 | 1.16.6 116 | 117 | 118 | 119 | org.javatuples 120 | javatuples 121 | 1.2 122 | 123 | 124 | 125 | 126 | 127 | junit 128 | junit 129 | 4.12 130 | test 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | org.apache.maven.plugins 140 | maven-compiler-plugin 141 | 3.3 142 | 143 | ${project.build.sourceEncoding} 144 | ${jdk.version} 145 | ${jdk.version} 146 | 147 | 148 | 149 | 150 | 151 | org.apache.maven.plugins 152 | maven-failsafe-plugin 153 | 2.19 154 | 155 | 156 | 157 | integration-test 158 | verify 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | org.apache.maven.plugins 167 | maven-assembly-plugin 168 | 2.4.1 169 | 170 | 171 | 172 | src/main/assembly/jar.xml 173 | src/main/assembly/dist.xml 174 | 175 | 176 | ${skipAssembly} 177 | 178 | 179 | 180 | 181 | make-assembly 182 | 183 | package 184 | 185 | single 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | org.apache.maven.plugins 194 | maven-source-plugin 195 | 2.2.1 196 | 197 | ${skipSource} 198 | 199 | 200 | 201 | attach-sources 202 | verify 203 | 204 | jar-no-fork 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | org.apache.maven.plugins 213 | maven-javadoc-plugin 214 | 2.9.1 215 | 216 | ${skipJavadoc} 217 | 218 | 219 | 220 | attach-javadocs 221 | verify 222 | 223 | jar 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | org.apache.maven.plugins 232 | maven-gpg-plugin 233 | 1.4 234 | 235 | ${skipSigning} 236 | 237 | 238 | 239 | sign-artifacts 240 | verify 241 | 242 | sign 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | org.apache.maven.plugins 251 | maven-release-plugin 252 | 2.5.3 253 | 254 | forked-path 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | org.codehaus.mojo 265 | findbugs-maven-plugin 266 | 2.5.2 267 | 268 | 269 | 270 | org.codehaus.mojo 271 | jdepend-maven-plugin 272 | 2.0-beta-2 273 | 274 | 275 | 276 | 277 | 278 | 279 | sonatype-nexus-staging 280 | Staging Releases 281 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 282 | 283 | 284 | sonatype-nexus-snapshots 285 | Snapshots Deployments 286 | https://oss.sonatype.org/content/repositories/snapshots/ 287 | 288 | 289 | 290 | 291 | 292 | 293 | develop 294 | 295 | true 296 | 297 | 298 | 299 | 300 | release 301 | 302 | false 303 | false 304 | false 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------