├── core ├── src │ ├── main │ │ ├── java │ │ │ ├── META-INF │ │ │ │ └── MANIFEST.MF │ │ │ └── com │ │ │ │ └── github │ │ │ │ └── ftrossbach │ │ │ │ └── kiqr │ │ │ │ └── core │ │ │ │ ├── query │ │ │ │ ├── exceptions │ │ │ │ │ ├── ScalarValueNotFoundException.java │ │ │ │ │ └── SerdeNotFoundException.java │ │ │ │ ├── AbstractKiqrVerticle.java │ │ │ │ ├── session │ │ │ │ │ └── SessionWindowQueryVerticle.java │ │ │ │ ├── kv │ │ │ │ │ ├── KeyValueCountVerticle.java │ │ │ │ │ ├── ScalarKeyValueQueryVerticle.java │ │ │ │ │ ├── AllKeyValuesQueryVerticle.java │ │ │ │ │ └── RangeKeyValueQueryVerticle.java │ │ │ │ ├── KiqrCodec.java │ │ │ │ ├── windowed │ │ │ │ │ └── WindowedQueryVerticle.java │ │ │ │ ├── facade │ │ │ │ │ ├── KeyBasedQueryFacadeVerticle.java │ │ │ │ │ └── ScatterGatherQueryFacadeVerticle.java │ │ │ │ └── AbstractQueryVerticle.java │ │ │ │ └── ShareableStreamsMetadataProvider.java │ │ └── resources │ │ │ └── vertx-default-jul-logging.properties.bak │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── ftrossbach │ │ └── kiqr │ │ └── core │ │ ├── query │ │ ├── SimpleWindowStoreIterator.java │ │ ├── SimpleKeyValueIterator.java │ │ ├── SimpleSessionIterator.java │ │ ├── KiqrCodecTest.java │ │ ├── kv │ │ │ └── KeyValueCountVerticleTest.java │ │ └── facade │ │ │ ├── KeyValueQueryFacadeVerticleTest.java │ │ │ ├── WindowQueryFacadeVerticleTest.java │ │ │ └── AllKeyValueQueryFacadeVerticleTest.java │ │ └── RuntimeVerticleTest.java └── pom.xml ├── commons ├── src │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── ftrossbach │ │ └── kiqr │ │ └── commons │ │ └── config │ │ ├── querymodel │ │ └── requests │ │ │ ├── HasKey.java │ │ │ ├── HasStoreName.java │ │ │ ├── KeyValueStoreCountQuery.java │ │ │ ├── StoreWideQuery.java │ │ │ ├── KeyBasedQuery.java │ │ │ ├── AllInstancesResponse.java │ │ │ ├── SessionQueryResponse.java │ │ │ ├── WindowedQueryResponse.java │ │ │ ├── ScalarKeyValueQueryResponse.java │ │ │ ├── RangeKeyValueQuery.java │ │ │ ├── WindowedQuery.java │ │ │ ├── MultiValuedKeyValueQueryResponse.java │ │ │ ├── AbstractQuery.java │ │ │ └── Window.java │ │ └── Config.java └── pom.xml ├── rest-client ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── ftrossbach │ │ │ └── kiqr │ │ │ └── client │ │ │ └── service │ │ │ ├── rest │ │ │ ├── MappingFunction.java │ │ │ ├── URISupplier.java │ │ │ └── SpecificBlockingRestKiqrClientImpl.java │ │ │ ├── SpecificBlockingKiqrClient.java │ │ │ ├── QueryExecutionException.java │ │ │ ├── ConnectionException.java │ │ │ └── GenericBlockingKiqrClient.java │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── ftrossbach │ │ └── kiqr │ │ └── client │ │ └── service │ │ └── rest │ │ ├── DummyVerticle.java │ │ ├── MockedRuntimeHttpServerVerticle.java │ │ ├── SpecificBlockingRestKiqrClientImplTest.java │ │ ├── CountQueryBlockingRestKiqrServiceImplTest.java │ │ ├── SessionWindowQueryBlockingRestKiqrServiceImplTest.java │ │ ├── ScalarKVQueryBlockingRestKiqrServiceImplTest.java │ │ ├── WindowQueryBlockingRestKiqrServiceImplTest.java │ │ └── AllKVQueryBlockingRestKiqrServiceImplTest.java └── pom.xml ├── .travis.yml ├── docker-compose.yml ├── APACHE-2.txt ├── rest-server ├── src │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── ftrossbach │ │ └── kiqr │ │ └── rest │ │ └── server │ │ ├── DummySuccessfulVerticle.java │ │ ├── DummyFailingVerticle.java │ │ ├── GenericHttpServerTest.java │ │ ├── CountQueryHttpServerTest.java │ │ ├── SessionWindowQueryHttpServerTest.java │ │ ├── ScalarQueryHttpServerTest.java │ │ ├── WindowedQueryHttpServerTest.java │ │ └── MultiValuedQueryHttpServerTest.java └── pom.xml ├── examples ├── src │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── ftrossbach │ │ └── kiqr │ │ └── examples │ │ ├── TestDriver.java │ │ ├── RestClient.java │ │ └── MainVerticle.java └── pom.xml ├── .gitignore └── README.md /core/src/main/java/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Main-Class: com.github.ftrossbach.kiqr.core.Main 3 | 4 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/HasKey.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 2 | 3 | /** 4 | * Created by ftr on 15/03/2017. 5 | */ 6 | public interface HasKey { 7 | 8 | byte[] getKey(); 9 | } 10 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/HasStoreName.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 2 | 3 | /** 4 | * Created by ftr on 15/03/2017. 5 | */ 6 | public interface HasStoreName { 7 | 8 | String getStoreName(); 9 | } 10 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/rest/MappingFunction.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.client.service.rest; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * Created by ftr on 10/03/2017. 7 | */ 8 | public interface MappingFunction { 9 | 10 | U apply(T t) throws IOException; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/exceptions/ScalarValueNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core.query.exceptions; 2 | 3 | /** 4 | * Created by ftr on 10/03/2017. 5 | */ 6 | public class ScalarValueNotFoundException extends RuntimeException{ 7 | public ScalarValueNotFoundException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | 5 | sudo: required 6 | 7 | services: 8 | - docker 9 | 10 | notifications: 11 | email: 12 | on_success: never # default: change 13 | on_failure: always # default: always 14 | 15 | before_install: 16 | # - docker-compose up -d 17 | 18 | after_success: 19 | - mvn clean test jacoco:report coveralls:report 20 | 21 | env: 22 | - KAFKA_PORT=29092 -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/rest/URISupplier.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.client.service.rest; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | /** 7 | * Created by ftr on 10/03/2017. 8 | */ 9 | @FunctionalInterface 10 | public interface URISupplier { 11 | 12 | 13 | T get() throws URISyntaxException, IOException; 14 | } 15 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2' 3 | services: 4 | zookeeper: 5 | image: confluentinc/cp-zookeeper:latest 6 | network_mode: host 7 | environment: 8 | ZOOKEEPER_CLIENT_PORT: 32181 9 | ZOOKEEPER_TICK_TIME: 2000 10 | 11 | kafka: 12 | image: confluentinc/cp-kafka:latest 13 | network_mode: host 14 | depends_on: 15 | - zookeeper 16 | environment: 17 | KAFKA_BROKER_ID: 1 18 | KAFKA_ZOOKEEPER_CONNECT: localhost:32181 19 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092 20 | KAFKA_NUM_PARTITIONS: 2 -------------------------------------------------------------------------------- /APACHE-2.txt: -------------------------------------------------------------------------------- 1 | Copyright © ${project.inceptionYear} ${owner} (${email}) 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /core/src/main/resources/vertx-default-jul-logging.properties.bak: -------------------------------------------------------------------------------- 1 | handlers=java.util.logging.ConsoleHandler,java.util.logging.FileHandler 2 | java.util.logging.SimpleFormatter.format=%5$s %6$s\n 3 | java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter 4 | java.util.logging.ConsoleHandler.level=FINEST 5 | java.util.logging.FileHandler.level=INFO 6 | java.util.logging.FileHandler.formatter=io.vertx.core.logging.impl.VertxLoggerFormatter 7 | 8 | # Put the log in the system temporary directory 9 | java.util.logging.FileHandler.pattern=%t/vertx.log 10 | 11 | .level=FINEST 12 | io.vertx.ext.web.level=FINEST 13 | io.vertx.level=FINEST 14 | com.hazelcast.level=FINEST 15 | io.netty.util.internal.PlatformDependent.level=FINEST -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/KeyValueStoreCountQuery.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 2 | 3 | /** 4 | * Created by ftr on 16/03/2017. 5 | */ 6 | public class KeyValueStoreCountQuery implements HasStoreName{ 7 | 8 | private String storeName; 9 | 10 | public KeyValueStoreCountQuery(String storeName) { 11 | this.storeName = storeName; 12 | } 13 | 14 | public KeyValueStoreCountQuery() { 15 | } 16 | 17 | public void setStoreName(String storeName) { 18 | this.storeName = storeName; 19 | } 20 | 21 | @Override 22 | public String getStoreName() { 23 | return storeName; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/SpecificBlockingKiqrClient.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.client.service; 2 | 3 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.Window; 4 | import org.apache.kafka.common.serialization.Serde; 5 | import java.util.Map; 6 | import java.util.Optional; 7 | 8 | /** 9 | * Created by ftr on 10/03/2017. 10 | */ 11 | public interface SpecificBlockingKiqrClient { 12 | 13 | Optional getScalarKeyValue(K key); 14 | 15 | Map getAllKeyValues(); 16 | 17 | Map getRangeKeyValues(K from, K to); 18 | 19 | Map getWindow(K key, long from, long to); 20 | 21 | Optional count(String store); 22 | 23 | Map getSession(K key); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/DummyVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import io.vertx.core.AbstractVerticle; 19 | 20 | /** 21 | * Created by ftr on 07/03/2017. 22 | */ 23 | public class DummyVerticle extends AbstractVerticle { 24 | } 25 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/DummySuccessfulVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import io.vertx.core.AbstractVerticle; 19 | import io.vertx.core.Future; 20 | 21 | /** 22 | * Created by ftr on 06/03/2017. 23 | */ 24 | public class DummySuccessfulVerticle extends AbstractVerticle { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/QueryExecutionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service; 17 | 18 | /** 19 | * Created by ftr on 07/03/2017. 20 | */ 21 | public class QueryExecutionException extends RuntimeException { 22 | public QueryExecutionException(String message) { 23 | super(message); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/ConnectionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service; 17 | 18 | /** 19 | * Created by ftr on 07/03/2017. 20 | */ 21 | public class ConnectionException extends RuntimeException { 22 | public ConnectionException(String message, Throwable cause) { 23 | super(message, cause); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/DummyFailingVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import io.vertx.core.AbstractVerticle; 19 | import io.vertx.core.Future; 20 | 21 | /** 22 | * Created by ftr on 06/03/2017. 23 | */ 24 | public class DummyFailingVerticle extends AbstractVerticle { 25 | 26 | @Override 27 | public void start(Future startFuture) throws Exception { 28 | startFuture.fail("oops"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/StoreWideQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public class StoreWideQuery extends AbstractQuery{ 22 | 23 | 24 | 25 | public StoreWideQuery() { 26 | } 27 | 28 | public StoreWideQuery(String storeName, String keySerde, String valueSerde) { 29 | super(storeName, keySerde, valueSerde); 30 | } 31 | 32 | 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/exceptions/SerdeNotFoundException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.exceptions; 17 | 18 | /** 19 | * Created by ftr on 03/03/2017. 20 | */ 21 | public class SerdeNotFoundException extends RuntimeException{ 22 | 23 | private final String serde; 24 | 25 | public SerdeNotFoundException(String serde, Throwable cause) { 26 | super("Serde not instantiable: " + serde, cause); 27 | this.serde = serde; 28 | } 29 | 30 | public String getSerde() { 31 | return serde; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/MockedRuntimeHttpServerVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | 19 | import com.github.ftrossbach.kiqr.rest.server.RestKiqrServerVerticle; 20 | import io.vertx.core.AbstractVerticle; 21 | import io.vertx.core.http.HttpServerOptions; 22 | 23 | /** 24 | * Created by ftr on 07/03/2017. 25 | */ 26 | public class MockedRuntimeHttpServerVerticle extends RestKiqrServerVerticle { 27 | public MockedRuntimeHttpServerVerticle(HttpServerOptions serverOptions, AbstractVerticle runtimeVerticle) { 28 | super(serverOptions, runtimeVerticle); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /commons/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | com.github.ftrossbach 23 | kiqr 24 | 0.0.2-SNAPSHOT 25 | 26 | 27 | kiqr-commons 28 | kiqr 29 | Common classes for KIQR 30 | https://github.com/ftrossbach/kiqr 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/KeyBasedQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public class KeyBasedQuery extends AbstractQuery implements HasKey{ 22 | 23 | private byte[] key; 24 | 25 | public KeyBasedQuery(){} 26 | 27 | public KeyBasedQuery(String storeName, String keySerde, byte[] key, String valueSerde) { 28 | super(storeName, keySerde, valueSerde); 29 | this.key = key; 30 | } 31 | 32 | public byte[] getKey() { 33 | return key; 34 | } 35 | 36 | public void setKey(byte[] key) { 37 | this.key = key; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/AllInstancesResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | import java.util.Set; 19 | 20 | /** 21 | * Created by ftr on 22/02/2017. 22 | */ 23 | public class AllInstancesResponse { 24 | private Set instances; 25 | 26 | 27 | public AllInstancesResponse() { 28 | } 29 | 30 | public AllInstancesResponse(Set instances) { 31 | this.instances = instances; 32 | } 33 | 34 | public Set getInstances() { 35 | return instances; 36 | } 37 | 38 | public void setInstances(Set instances) { 39 | this.instances = instances; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/SessionQueryResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | 19 | 20 | import java.util.List; 21 | import java.util.SortedMap; 22 | 23 | /** 24 | * Created by ftr on 20/02/2017. 25 | */ 26 | public class SessionQueryResponse { 27 | 28 | 29 | 30 | private List values; 31 | 32 | public SessionQueryResponse() { 33 | 34 | } 35 | 36 | public SessionQueryResponse(Listvalues) { 37 | 38 | this.values = values; 39 | } 40 | 41 | public List getValues() { 42 | return values; 43 | } 44 | 45 | public void setValues(List values) { 46 | this.values = values; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/WindowedQueryResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | 19 | 20 | import java.util.SortedMap; 21 | 22 | /** 23 | * Created by ftr on 20/02/2017. 24 | */ 25 | public class WindowedQueryResponse { 26 | 27 | 28 | 29 | private SortedMap values; 30 | 31 | public WindowedQueryResponse() { 32 | 33 | } 34 | 35 | public WindowedQueryResponse(SortedMap values) { 36 | 37 | this.values = values; 38 | } 39 | 40 | public void setValues(SortedMap values) { 41 | this.values = values; 42 | } 43 | 44 | public SortedMap getValues() { 45 | return values; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/ScalarKeyValueQueryResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public class ScalarKeyValueQueryResponse { 22 | 23 | private String value; 24 | 25 | public ScalarKeyValueQueryResponse(){} 26 | 27 | public ScalarKeyValueQueryResponse(String value) { 28 | 29 | this.value = value; 30 | } 31 | 32 | public String getValue() { 33 | return value; 34 | } 35 | 36 | public void setValue(String value) { 37 | this.value = value; 38 | } 39 | 40 | 41 | @Override 42 | public String toString() { 43 | return "ScalarKeyValueQueryResponse{" + 44 | "value=" + value + 45 | '}'; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/RangeKeyValueQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public class RangeKeyValueQuery extends AbstractQuery{ 22 | 23 | 24 | private byte[] from; 25 | private byte[] to; 26 | 27 | public RangeKeyValueQuery() { 28 | } 29 | 30 | public RangeKeyValueQuery(String storeName, String keySerde, String valueSerde, byte[] from, byte[] to) { 31 | super(storeName, keySerde, valueSerde); 32 | this.from = from; 33 | this.to = to; 34 | } 35 | 36 | public byte[] getFrom() { 37 | return from; 38 | } 39 | 40 | public void setFrom(byte[] from) { 41 | this.from = from; 42 | } 43 | 44 | public byte[] getTo() { 45 | return to; 46 | } 47 | 48 | public void setTo(byte[] to) { 49 | this.to = to; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/SimpleWindowStoreIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import org.apache.kafka.streams.KeyValue; 19 | import org.apache.kafka.streams.state.WindowStoreIterator; 20 | import java.util.Arrays; 21 | import java.util.Iterator; 22 | 23 | /** 24 | * Created by ftr on 05/03/2017. 25 | */ 26 | public class SimpleWindowStoreIterator implements WindowStoreIterator{ 27 | 28 | private final Iterator> iterator; 29 | public boolean closed = false; 30 | 31 | public SimpleWindowStoreIterator(KeyValue... values){ 32 | iterator = Arrays.asList(values).iterator(); 33 | } 34 | 35 | @Override 36 | public void close() { 37 | closed = true; 38 | } 39 | 40 | @Override 41 | public Long peekNextKey() { 42 | throw new UnsupportedOperationException(); 43 | } 44 | 45 | @Override 46 | public boolean hasNext() { 47 | return iterator.hasNext(); 48 | } 49 | 50 | @Override 51 | public KeyValue next() { 52 | return iterator.next(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /examples/src/main/java/com/github/ftrossbach/kiqr/examples/TestDriver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.examples; 17 | 18 | import org.apache.kafka.clients.producer.KafkaProducer; 19 | import org.apache.kafka.clients.producer.ProducerRecord; 20 | 21 | import java.util.Properties; 22 | 23 | /** 24 | * Created by ftr on 17/02/2017. 25 | */ 26 | public class TestDriver { 27 | 28 | public static void main(String[] args) { 29 | Properties props = new Properties(); 30 | props.put("bootstrap.servers", "localhost:9092"); 31 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 32 | props.put("value.serializer", "org.apache.kafka.common.serialization.LongSerializer"); 33 | props.put("linger.ms", 0); 34 | 35 | KafkaProducer producer = new KafkaProducer<>(props); 36 | 37 | for(int i=0; i < 10000; i++){ 38 | String ip = "127.0.0." + i % 10; 39 | System.out.println(ip); 40 | producer.send(new ProducerRecord<>("visits", ip, System.currentTimeMillis() + i)); 41 | } 42 | 43 | producer.close(); 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/WindowedQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public class WindowedQuery extends AbstractQuery implements HasKey{ 22 | 23 | private byte[] key; 24 | private long from; 25 | private long to; 26 | 27 | public WindowedQuery(){} 28 | 29 | public WindowedQuery(String storeName, String keySerde, byte[] key, String valueSerde, long from, long to) { 30 | super(storeName, keySerde, valueSerde); 31 | this.from = from; 32 | this.to = to; 33 | this.key = key; 34 | } 35 | 36 | public long getFrom() { 37 | return from; 38 | } 39 | 40 | public void setFrom(long from) { 41 | this.from = from; 42 | } 43 | 44 | public long getTo() { 45 | return to; 46 | } 47 | 48 | public void setTo(long to) { 49 | this.to = to; 50 | } 51 | 52 | public byte[] getKey() { 53 | return key; 54 | } 55 | 56 | public void setKey(byte[] key) { 57 | this.key = key; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/SimpleKeyValueIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import org.apache.kafka.streams.KeyValue; 19 | import org.apache.kafka.streams.state.KeyValueIterator; 20 | import java.util.Arrays; 21 | import java.util.Iterator; 22 | 23 | /** 24 | * Created by ftr on 05/03/2017. 25 | */ 26 | public class SimpleKeyValueIterator implements KeyValueIterator { 27 | 28 | 29 | public boolean closed = false; 30 | 31 | private final Iterator> iterator; 32 | 33 | public SimpleKeyValueIterator(KeyValue... values){ 34 | 35 | iterator = Arrays.asList(values).iterator(); 36 | 37 | } 38 | 39 | @Override 40 | public void close() { 41 | closed = true; 42 | } 43 | 44 | @Override 45 | public Object peekNextKey() { 46 | throw new UnsupportedOperationException(); 47 | } 48 | 49 | @Override 50 | public boolean hasNext() { 51 | return iterator.hasNext(); 52 | } 53 | 54 | @Override 55 | public KeyValue next() { 56 | return iterator.next(); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/MultiValuedKeyValueQueryResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | /** 24 | * Created by ftr on 20/02/2017. 25 | */ 26 | public class MultiValuedKeyValueQueryResponse{ 27 | 28 | 29 | private Map results = new HashMap<>(); 30 | 31 | public MultiValuedKeyValueQueryResponse() { 32 | } 33 | 34 | public MultiValuedKeyValueQueryResponse(Map results) { 35 | this.results = results; 36 | } 37 | 38 | public Map getResults() { 39 | return results; 40 | } 41 | 42 | public void setResults(Map results) { 43 | this.results = results; 44 | } 45 | 46 | public MultiValuedKeyValueQueryResponse merge(MultiValuedKeyValueQueryResponse other){ 47 | 48 | Map left = new HashMap<>(this.results); 49 | Map right = new HashMap<>(other.results); 50 | left.putAll(right); 51 | 52 | return new MultiValuedKeyValueQueryResponse(left); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/AbstractQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 17 | 18 | /** 19 | * Created by ftr on 20/02/2017. 20 | */ 21 | public abstract class AbstractQuery implements HasStoreName{ 22 | 23 | private String storeName; 24 | private String keySerde; 25 | private String valueSerde; 26 | 27 | 28 | public AbstractQuery(){} 29 | 30 | public AbstractQuery(String storeName, String keySerde, String valueSerde) { 31 | this.storeName = storeName; 32 | this.keySerde = keySerde; 33 | 34 | this.valueSerde = valueSerde; 35 | } 36 | 37 | public String getStoreName() { 38 | return storeName; 39 | } 40 | 41 | public void setStoreName(String storeName) { 42 | this.storeName = storeName; 43 | } 44 | 45 | public String getKeySerde() { 46 | return keySerde; 47 | } 48 | 49 | public void setKeySerde(String keySerde) { 50 | this.keySerde = keySerde; 51 | } 52 | 53 | public String getValueSerde() { 54 | return valueSerde; 55 | } 56 | 57 | public void setValueSerde(String valueSerde) { 58 | this.valueSerde = valueSerde; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/GenericBlockingKiqrClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.Window; 19 | import org.apache.kafka.common.serialization.Serde; 20 | 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | /** 25 | * Created by ftr on 01/03/2017. 26 | */ 27 | public interface GenericBlockingKiqrClient { 28 | 29 | Optional getScalarKeyValue(String store, Class keyClass, K key, Class valueClass, Serde keySerde,Serde valueSerde); 30 | 31 | Map getAllKeyValues(String store, Class keyClass, Class valueClass, Serde keySerde,Serde valueSerde); 32 | 33 | Map getRangeKeyValues(String store, Class keyClass, Class valueClass, Serde keySerde,Serde valueSerde, K from, K to); 34 | 35 | Map getWindow(String store, Class keyClass, K key, Class valueClass,Serde keySerde,Serde valueSerde, long from, long to); 36 | 37 | Optional count(String store); 38 | 39 | Map getSession(String store, Class keyClass, K key, Class valueClass, Serde keySerde,Serde valueSerde); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/SimpleSessionIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import org.apache.kafka.streams.KeyValue; 19 | import org.apache.kafka.streams.kstream.Windowed; 20 | import org.apache.kafka.streams.state.KeyValueIterator; 21 | import java.util.Arrays; 22 | import java.util.Iterator; 23 | 24 | /** 25 | * Created by ftr on 05/03/2017. 26 | */ 27 | public class SimpleSessionIterator implements KeyValueIterator, Object> { 28 | 29 | 30 | public boolean closed = false; 31 | 32 | private final Iterator, Object>> iterator; 33 | 34 | public SimpleSessionIterator(KeyValue, Object>... values){ 35 | 36 | iterator = Arrays.asList(values).iterator(); 37 | 38 | } 39 | 40 | @Override 41 | public void close() { 42 | closed = true; 43 | } 44 | 45 | @Override 46 | public Windowed peekNextKey() { 47 | throw new UnsupportedOperationException(); 48 | } 49 | 50 | @Override 51 | public boolean hasNext() { 52 | return iterator.hasNext(); 53 | } 54 | 55 | @Override 56 | public KeyValue, Object> next() { 57 | return iterator.next(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/querymodel/requests/Window.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.commons.config.querymodel.requests; 2 | 3 | /** 4 | * Created by ftr on 16/03/2017. 5 | */ 6 | public class Window implements Comparable{ 7 | 8 | private long startMs; 9 | private long endMs; 10 | private String value; 11 | 12 | 13 | public Window(long startMs, long endMs, String value) { 14 | this.startMs = startMs; 15 | this.endMs = endMs; 16 | this.value = value; 17 | } 18 | 19 | public Window() { 20 | } 21 | 22 | public long getStartMs() { 23 | return startMs; 24 | } 25 | 26 | public void setStartMs(long startMs) { 27 | this.startMs = startMs; 28 | } 29 | 30 | public long getEndMs() { 31 | return endMs; 32 | } 33 | 34 | public void setEndMs(long endMs) { 35 | this.endMs = endMs; 36 | } 37 | 38 | public String getValue() { 39 | return value; 40 | } 41 | 42 | public void setValue(String value) { 43 | this.value = value; 44 | } 45 | 46 | @Override 47 | public int compareTo(Window o) { 48 | return Long.valueOf(startMs).compareTo(endMs); 49 | } 50 | 51 | @Override 52 | public boolean equals(Object o) { 53 | if (this == o) return true; 54 | if (o == null || getClass() != o.getClass()) return false; 55 | 56 | Window window = (Window) o; 57 | 58 | if (startMs != window.startMs) return false; 59 | return endMs == window.endMs; 60 | } 61 | 62 | @Override 63 | public int hashCode() { 64 | int result = (int) (startMs ^ (startMs >>> 32)); 65 | result = 31 * result + (int) (endMs ^ (endMs >>> 32)); 66 | return result; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "Window{" + 72 | "startMs=" + startMs + 73 | ", endMs=" + endMs + 74 | '}'; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/ShareableStreamsMetadataProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core; 2 | 3 | import io.vertx.core.shareddata.Shareable; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.common.serialization.Serializer; 6 | import org.apache.kafka.streams.KafkaStreams; 7 | import org.apache.kafka.streams.StreamsConfig; 8 | import org.apache.kafka.streams.kstream.KStream; 9 | import org.apache.kafka.streams.kstream.KStreamBuilder; 10 | import org.apache.kafka.streams.kstream.KTable; 11 | import org.apache.kafka.streams.processor.StateStore; 12 | import org.apache.kafka.streams.processor.StreamPartitioner; 13 | import org.apache.kafka.streams.state.StreamsMetadata; 14 | import java.util.Collection; 15 | 16 | /** 17 | * Created by ftr on 15/03/2017. 18 | */ 19 | public class ShareableStreamsMetadataProvider implements Shareable { 20 | 21 | private final KafkaStreams streamsClient; 22 | 23 | public ShareableStreamsMetadataProvider(KafkaStreams streamsClient) { 24 | this.streamsClient = streamsClient; 25 | } 26 | 27 | 28 | public Collection allMetadata() { 29 | return streamsClient.allMetadata(); 30 | } 31 | 32 | 33 | public Collection allMetadataForStore(final String storeName) { 34 | return streamsClient.allMetadataForStore(storeName); 35 | } 36 | 37 | public StreamsMetadata metadataForKey(final String storeName, 38 | final K key, 39 | final Serializer keySerializer) { 40 | 41 | return streamsClient.metadataForKey(storeName, key, keySerializer); 42 | } 43 | 44 | public StreamsMetadata metadataForKey(final String storeName, 45 | final K key, 46 | final StreamPartitioner partitioner) { 47 | return streamsClient.metadataForKey(storeName, key, partitioner); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/AbstractKiqrVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import com.github.ftrossbach.kiqr.core.query.exceptions.SerdeNotFoundException; 19 | import io.vertx.core.AbstractVerticle; 20 | import org.apache.kafka.common.serialization.Serde; 21 | import org.apache.kafka.streams.KafkaStreams; 22 | 23 | import java.util.Base64; 24 | 25 | /** 26 | * Created by ftr on 19/02/2017. 27 | */ 28 | public abstract class AbstractKiqrVerticle extends AbstractVerticle{ 29 | 30 | 31 | 32 | protected Serde getSerde(String serde){ 33 | try { 34 | return (Serde) Class.forName(serde).newInstance(); 35 | 36 | } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | ClassCastException e) { 37 | throw new SerdeNotFoundException(serde, e); 38 | } 39 | } 40 | 41 | protected Object deserializeObject(Serde serde, byte[] key){ 42 | return serde.deserializer().deserialize("?", key); 43 | } 44 | 45 | protected byte[] serializeObject(Serde serde, Object obj) { 46 | return serde.serializer().serialize("?", obj); 47 | } 48 | 49 | protected String base64Encode(Serde serde, Object obj){ 50 | return Base64.getEncoder().encodeToString(serializeObject(serde, obj)); 51 | } 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/session/SessionWindowQueryVerticle.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core.query.session; 2 | 3 | import com.github.ftrossbach.kiqr.commons.config.Config; 4 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 5 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 6 | import org.apache.kafka.streams.KafkaStreams; 7 | import org.apache.kafka.streams.KeyValue; 8 | import org.apache.kafka.streams.kstream.Windowed; 9 | import org.apache.kafka.streams.state.*; 10 | import java.util.*; 11 | 12 | /** 13 | * Created by ftr on 16/03/2017. 14 | */ 15 | public class SessionWindowQueryVerticle extends AbstractQueryVerticle { 16 | public SessionWindowQueryVerticle(String instanceId, KafkaStreams streams) { 17 | super(instanceId, streams); 18 | } 19 | 20 | @Override 21 | public void start() throws Exception { 22 | 23 | execute(Config.SESSION_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> { 24 | 25 | KeyBasedQuery query = (KeyBasedQuery) abstractQuery; 26 | ReadOnlySessionStore store = streams.store(query.getStoreName(), QueryableStoreTypes.sessionStore()); 27 | try (KeyValueIterator, Object> result = store.fetch(deserializeObject(keySerde, query.getKey()))) { 28 | 29 | if (result.hasNext()) { 30 | List results = new ArrayList<>(); 31 | while (result.hasNext()) { 32 | 33 | 34 | KeyValue, Object> windowedEntry = result.next(); 35 | results.add(new Window(windowedEntry.key.window().start(), windowedEntry.key.window().end(), base64Encode(valueSerde, windowedEntry.value))); 36 | } 37 | return new SessionQueryResponse(results); 38 | } else { 39 | return new SessionQueryResponse(Collections.emptyList()); 40 | } 41 | } 42 | }); 43 | 44 | 45 | } 46 | } -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/kv/KeyValueCountVerticle.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core.query.kv; 2 | 3 | import com.github.ftrossbach.kiqr.commons.config.Config; 4 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyValueStoreCountQuery; 5 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 6 | import io.vertx.core.logging.Logger; 7 | import io.vertx.core.logging.LoggerFactory; 8 | import org.apache.kafka.streams.KafkaStreams; 9 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 10 | import org.apache.kafka.streams.state.QueryableStoreTypes; 11 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 12 | 13 | /** 14 | * Created by ftr on 15/03/2017. 15 | */ 16 | public class KeyValueCountVerticle extends AbstractQueryVerticle{ 17 | private static final Logger LOG = LoggerFactory.getLogger(KeyValueCountVerticle.class); 18 | 19 | 20 | public KeyValueCountVerticle(String instanceId, KafkaStreams streams) { 21 | super(instanceId, streams); 22 | } 23 | 24 | @Override 25 | public void start() throws Exception { 26 | 27 | vertx.eventBus().consumer(Config.COUNT_KEY_VALUE_QUERY_ADDRESS_PREFIX + instanceId, msg -> { 28 | KeyValueStoreCountQuery query = (KeyValueStoreCountQuery) msg.body(); 29 | try { 30 | ReadOnlyKeyValueStore kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore()); 31 | long count = kvStore.approximateNumEntries(); 32 | msg.reply(count); 33 | } catch (InvalidStateStoreException e) { 34 | LOG.error(String.format("Store %s not queriable. Could be due to wrong store type or rebalancing", query.getStoreName())); 35 | msg.fail(500, String.format("Store not available due to rebalancing or wrong store type")); 36 | } catch (RuntimeException e) { 37 | LOG.error("Unexpected exception", e); 38 | msg.fail(500, e.getMessage()); 39 | } 40 | }); 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/KiqrCodec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import io.vertx.core.buffer.Buffer; 19 | import io.vertx.core.eventbus.MessageCodec; 20 | import io.vertx.core.eventbus.impl.codecs.JsonObjectMessageCodec; 21 | import io.vertx.core.eventbus.impl.codecs.StringMessageCodec; 22 | import io.vertx.core.json.Json; 23 | import io.vertx.core.json.JsonObject; 24 | 25 | /** 26 | * Created by ftr on 20/02/2017. 27 | */ 28 | public class KiqrCodec implements MessageCodec { 29 | 30 | private final JsonObjectMessageCodec codec = new JsonObjectMessageCodec(); 31 | 32 | private final Class clazz; 33 | 34 | public KiqrCodec(Class clazz){ 35 | this.clazz = clazz; 36 | } 37 | 38 | 39 | @Override 40 | public void encodeToWire(Buffer buffer, T object) { 41 | //ToDo: more efficient serialization than JSON for internal purposes 42 | codec.encodeToWire(buffer, JsonObject.mapFrom(object)); 43 | } 44 | 45 | @Override 46 | public T decodeFromWire(int pos, Buffer buffer) { 47 | //ToDo: more efficient deserialization 48 | 49 | return codec.decodeFromWire(pos, buffer).mapTo(clazz); 50 | } 51 | 52 | 53 | @Override 54 | public T transform(T object) { 55 | return object; 56 | } 57 | 58 | @Override 59 | public String name() { 60 | return clazz.getName(); 61 | } 62 | 63 | @Override 64 | public byte systemCodecID() { 65 | return -1; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | 23 | com.github.ftrossbach 24 | kiqr 25 | 0.0.2-SNAPSHOT 26 | 27 | 28 | kiqr-core 29 | 30 | kiqr-core 31 | Core KIQR module 32 | https://github.com/ftrossbach/kiqr 33 | 34 | 35 | 36 | ${project.groupId} 37 | kiqr-commons 38 | 39 | 40 | io.vertx 41 | vertx-core 42 | 43 | 44 | com.fasterxml.jackson.datatype 45 | jackson-datatype-jdk8 46 | 47 | 48 | 49 | org.apache.kafka 50 | kafka-streams 51 | 52 | 53 | org.apache.commons 54 | commons-lang3 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /rest-server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | 23 | com.github.ftrossbach 24 | kiqr 25 | 0.0.2-SNAPSHOT 26 | 27 | 28 | kiqr-rest-server 29 | 30 | kiqr-rest-server 31 | REST-Server for KIQR 32 | https://github.com/ftrossbach/kiqr 33 | 34 | 35 | 36 | 37 | ${project.groupId} 38 | kiqr-commons 39 | 40 | 41 | 42 | ${project.groupId} 43 | kiqr-core 44 | 45 | 46 | io.vertx 47 | vertx-core 48 | 49 | 50 | io.vertx 51 | vertx-web 52 | 53 | 54 | 55 | org.apache.kafka 56 | kafka-streams 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /examples/src/main/java/com/github/ftrossbach/kiqr/examples/RestClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.examples; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.GenericBlockingKiqrClient; 19 | import com.github.ftrossbach.kiqr.client.service.rest.GenericBlockingRestKiqrClientImpl; 20 | import org.apache.kafka.common.serialization.Serdes; 21 | 22 | import java.time.Instant; 23 | import java.time.LocalDateTime; 24 | import java.time.ZoneId; 25 | import java.util.Map; 26 | 27 | /** 28 | * Created by ftr on 01/03/2017. 29 | */ 30 | public class RestClient { 31 | 32 | public static void main(String[] args) { 33 | GenericBlockingKiqrClient service = new GenericBlockingRestKiqrClientImpl("localhost", 2901); 34 | 35 | System.out.println(service.getScalarKeyValue("visitStore", String.class, "127.0.0.1", Long.class, Serdes.String(), Serdes.Long())); 36 | 37 | System.out.println(service.getAllKeyValues("visitStores", String.class, Long.class, Serdes.String(), Serdes.Long())); 38 | System.out.println(service.getRangeKeyValues("visitStore", String.class, Long.class, Serdes.String(), Serdes.Long(), "127.0.0.2", "127.0.0.5")); 39 | Map visitCount = service.getWindow("visitCount", String.class, "127.0.0.1", Long.class, Serdes.String(), Serdes.Long(), System.currentTimeMillis() - 1000 * 60 * 60, System.currentTimeMillis()); 40 | System.out.println(visitCount); 41 | 42 | visitCount.forEach((time, val) -> System.out.println( 43 | LocalDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneId.systemDefault()))); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/osx,maven,intellij+iml 3 | 4 | ### Intellij+iml ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff: 9 | .idea 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | 13 | # Sensitive or high-churn files: 14 | .idea/**/dataSources/ 15 | .idea/**/dataSources.ids 16 | .idea/**/dataSources.xml 17 | .idea/**/dataSources.local.xml 18 | .idea/**/sqlDataSources.xml 19 | .idea/**/dynamic.xml 20 | .idea/**/uiDesigner.xml 21 | 22 | # Gradle: 23 | .idea/**/gradle.xml 24 | .idea/**/libraries 25 | 26 | # Mongo Explorer plugin: 27 | .idea/**/mongoSettings.xml 28 | 29 | ## File-based project format: 30 | *.iws 31 | 32 | ## Plugin-specific files: 33 | 34 | # IntelliJ 35 | /out/ 36 | 37 | # mpeltonen/sbt-idea plugin 38 | .idea_modules/ 39 | 40 | # JIRA plugin 41 | atlassian-ide-plugin.xml 42 | 43 | # Crashlytics plugin (for Android Studio and IntelliJ) 44 | com_crashlytics_export_strings.xml 45 | crashlytics.properties 46 | crashlytics-build.properties 47 | fabric.properties 48 | 49 | ### Intellij+iml Patch ### 50 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 51 | 52 | *.iml 53 | modules.xml 54 | .idea/misc.xml 55 | *.ipr 56 | 57 | ### Maven ### 58 | target/ 59 | pom.xml.tag 60 | pom.xml.releaseBackup 61 | pom.xml.versionsBackup 62 | pom.xml.next 63 | release.properties 64 | dependency-reduced-pom.xml 65 | buildNumber.properties 66 | .mvn/timing.properties 67 | 68 | # Exclude maven wrapper 69 | !/.mvn/wrapper/maven-wrapper.jar 70 | 71 | ### OSX ### 72 | *.DS_Store 73 | .AppleDouble 74 | .LSOverride 75 | 76 | # Icon must end with two \r 77 | Icon 78 | 79 | 80 | # Thumbnails 81 | ._* 82 | 83 | # Files that might appear in the root of a volume 84 | .DocumentRevisions-V100 85 | .fseventsd 86 | .Spotlight-V100 87 | .TemporaryItems 88 | .Trashes 89 | .VolumeIcon.icns 90 | .com.apple.timemachine.donotpresent 91 | 92 | # Directories potentially created on remote AFP share 93 | .AppleDB 94 | .AppleDesktop 95 | Network Trash Folder 96 | Temporary Items 97 | .apdisk 98 | 99 | # End of https://www.gitignore.io/api/osx,maven,intellij+iml 100 | 0 101 | 102 | -------------------------------------------------------------------------------- /commons/src/main/java/com/github/ftrossbach/kiqr/commons/config/Config.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.commons.config; 17 | 18 | /** 19 | * Created by ftr on 19/02/2017. 20 | */ 21 | public class Config { 22 | 23 | 24 | 25 | public static String INSTANCE_RESOLVER_ADDRESS_SINGLE = "com.github.ftrossbach.kiqr.core.resolver.single"; 26 | public static String ALL_INSTANCES = "com.github.ftrossbach.kiqr.core.resolver.all"; 27 | 28 | public static String KEY_VALUE_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.kv."; 29 | public static String ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.all_kv."; 30 | public static String RANGE_KEY_VALUE_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.range_kv."; 31 | public static String COUNT_KEY_VALUE_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.count_kv."; 32 | public static String WINDOWED_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.window."; 33 | public static String SESSION_QUERY_ADDRESS_PREFIX = "com.github.ftrossbach.kiqr.core.query.session."; 34 | 35 | public static String KEY_VALUE_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.kv"; 36 | public static String ALL_KEY_VALUE_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.all_kv"; 37 | public static String RANGE_KEY_VALUE_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.range_kv"; 38 | public static String COUNT_KEY_VALUE_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.count"; 39 | public static String WINDOWED_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.window"; 40 | public static String SESSION_QUERY_FACADE_ADDRESS = "com.github.ftrossbach.kiqr.core.query.facade.session"; 41 | 42 | public static String CLUSTER_STATE_BROADCAST_ADDRESS = "com.github.ftrossbach.kiqr.core.cluster.state"; 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/kv/ScalarKeyValueQueryVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.kv; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyBasedQuery; 20 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.ScalarKeyValueQueryResponse; 21 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 22 | import com.github.ftrossbach.kiqr.core.query.exceptions.ScalarValueNotFoundException; 23 | import org.apache.kafka.streams.KafkaStreams; 24 | import org.apache.kafka.streams.state.QueryableStoreTypes; 25 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 26 | 27 | /** 28 | * Created by ftr on 19/02/2017. 29 | */ 30 | public class ScalarKeyValueQueryVerticle extends AbstractQueryVerticle { 31 | 32 | 33 | public ScalarKeyValueQueryVerticle(String instanceId, KafkaStreams streams) { 34 | super(instanceId, streams); 35 | } 36 | 37 | 38 | 39 | @Override 40 | public void start() throws Exception { 41 | 42 | execute(Config.KEY_VALUE_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> { 43 | KeyBasedQuery query = (KeyBasedQuery) abstractQuery; 44 | 45 | Object deserializedKey = deserializeObject(keySerde, query.getKey()); 46 | ReadOnlyKeyValueStore kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore()); 47 | Object result = kvStore.get(deserializedKey); 48 | if (result != null) { 49 | return new ScalarKeyValueQueryResponse(base64Encode(valueSerde, result)); 50 | } else { 51 | throw new ScalarValueNotFoundException(String.format("Key %s not found in store %s", deserializedKey.toString(), abstractQuery.getStoreName())); 52 | } 53 | }); 54 | 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /examples/src/main/java/com/github/ftrossbach/kiqr/examples/MainVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.examples; 17 | 18 | import com.github.ftrossbach.kiqr.rest.server.RestKiqrServerVerticle; 19 | import io.vertx.core.AbstractVerticle; 20 | import io.vertx.core.Future; 21 | import org.apache.kafka.common.serialization.Serdes; 22 | import org.apache.kafka.streams.StreamsConfig; 23 | import org.apache.kafka.streams.kstream.KStreamBuilder; 24 | import org.apache.kafka.streams.kstream.KTable; 25 | import org.apache.kafka.streams.kstream.TimeWindows; 26 | import org.apache.kafka.streams.kstream.Windowed; 27 | 28 | import java.util.Properties; 29 | 30 | /** 31 | * Created by ftr on 19/02/2017. 32 | */ 33 | public class MainVerticle extends AbstractVerticle { 34 | @Override 35 | public void start(Future startFuture) throws Exception { 36 | Properties props = new Properties(); 37 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "kiqr"); 38 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 39 | props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class); 40 | props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.LongSerde.class); 41 | props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0"); 42 | 43 | KStreamBuilder builder = new KStreamBuilder(); 44 | KTable table = builder.table(Serdes.String(), Serdes.Long(), "visits", "visitStore"); 45 | KTable, Long> windowedCount = table.toStream().groupByKey().count(TimeWindows.of(60), "visitCount"); 46 | 47 | 48 | 49 | 50 | 51 | vertx.deployVerticle(RestKiqrServerVerticle.Builder.serverBuilder(builder, props).withPort(2901).build(), res -> { 52 | if (res.succeeded()) { 53 | startFuture.complete(); 54 | } else { 55 | startFuture.fail(res.cause()); 56 | } 57 | }); 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/KiqrCodecTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyBasedQuery; 19 | import io.vertx.core.buffer.Buffer; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | 24 | import static org.junit.Assert.*; 25 | import static org.hamcrest.CoreMatchers.*; 26 | 27 | /** 28 | * Created by ftr on 06/03/2017. 29 | */ 30 | public class KiqrCodecTest { 31 | 32 | private KiqrCodec uut; 33 | 34 | @Before 35 | public void setUp(){ 36 | uut = new KiqrCodec<>(KeyBasedQuery.class); 37 | } 38 | 39 | @Test 40 | public void decodeEncode(){ 41 | 42 | Buffer buffer = Buffer.buffer(); 43 | uut.encodeToWire(buffer, new KeyBasedQuery("store", "key serde", "key".getBytes(), "value serde")); 44 | 45 | KeyBasedQuery deserializedResult = uut.decodeFromWire(0, buffer); 46 | 47 | assertThat(deserializedResult.getStoreName(), is(equalTo("store"))); 48 | assertThat(deserializedResult.getKeySerde(), is(equalTo("key serde"))); 49 | assertThat(new String(deserializedResult.getKey()), is(equalTo("key"))); 50 | assertThat(deserializedResult.getValueSerde(), is(equalTo("value serde"))); 51 | 52 | } 53 | 54 | @Test 55 | public void passThrough(){ 56 | 57 | KeyBasedQuery result = uut.transform(new KeyBasedQuery("store", "key serde", "key".getBytes(), "value serde")); 58 | assertThat(result.getStoreName(), is(equalTo("store"))); 59 | assertThat(result.getKeySerde(), is(equalTo("key serde"))); 60 | assertThat(new String(result.getKey()), is(equalTo("key"))); 61 | assertThat(result.getValueSerde(), is(equalTo("value serde"))); 62 | } 63 | 64 | @Test 65 | public void id(){ 66 | 67 | byte b = uut.systemCodecID(); 68 | 69 | assertThat(b, is(equalTo((byte) -1))); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/windowed/WindowedQueryVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | *

4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | *

8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | *

10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.windowed; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.exceptions.SerdeNotFoundException; 22 | import org.apache.kafka.common.serialization.Serde; 23 | import org.apache.kafka.streams.KafkaStreams; 24 | import org.apache.kafka.streams.KeyValue; 25 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 26 | import org.apache.kafka.streams.state.*; 27 | 28 | import java.util.*; 29 | 30 | /** 31 | * Created by ftr on 19/02/2017. 32 | */ 33 | public class WindowedQueryVerticle extends AbstractQueryVerticle { 34 | 35 | 36 | public WindowedQueryVerticle(String instanceId, KafkaStreams streams) { 37 | super(instanceId, streams); 38 | } 39 | 40 | @Override 41 | public void start() throws Exception { 42 | 43 | execute(Config.WINDOWED_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> { 44 | 45 | WindowedQuery query = (WindowedQuery) abstractQuery; 46 | ReadOnlyWindowStore windowStore = streams.store(query.getStoreName(), QueryableStoreTypes.windowStore()); 47 | 48 | try (WindowStoreIterator result = windowStore.fetch(deserializeObject(keySerde, query.getKey()), query.getFrom(), query.getTo())) { 49 | 50 | if (result.hasNext()) { 51 | SortedMap results = new TreeMap<>(); 52 | while (result.hasNext()) { 53 | KeyValue windowedEntry = result.next(); 54 | results.put(windowedEntry.key, base64Encode(valueSerde, windowedEntry.value)); 55 | } 56 | return new WindowedQueryResponse(results); 57 | } else { 58 | return new WindowedQueryResponse(Collections.emptySortedMap()); 59 | } 60 | } 61 | }); 62 | 63 | 64 | } 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/GenericHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import io.vertx.core.AbstractVerticle; 19 | import io.vertx.core.http.HttpServerOptions; 20 | import io.vertx.ext.unit.TestContext; 21 | import io.vertx.ext.unit.junit.RunTestOnContext; 22 | import io.vertx.ext.unit.junit.VertxUnitRunner; 23 | import org.apache.kafka.streams.kstream.KStreamBuilder; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | import org.junit.runner.RunWith; 27 | import java.util.Properties; 28 | import static org.junit.Assert.*; 29 | import static org.hamcrest.Matchers.*; 30 | 31 | /** 32 | * Created by ftr on 07/03/2017. 33 | */ 34 | @RunWith(VertxUnitRunner.class) 35 | public class GenericHttpServerTest { 36 | 37 | 38 | @Rule 39 | public final RunTestOnContext rule = new RunTestOnContext(); 40 | 41 | @Test 42 | public void deploymentFailure(TestContext context){ 43 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummyFailingVerticle()), context.asyncAssertFailure()); 44 | } 45 | 46 | @Test 47 | public void builderWithConfig(){ 48 | RestKiqrServerVerticle.Builder builder = RestKiqrServerVerticle.Builder.serverBuilder(new KStreamBuilder(), new Properties()); 49 | 50 | AbstractVerticle serverVerticle = builder.withOptions(new HttpServerOptions().setPort(4711)).build(); 51 | assertThat(serverVerticle, is(instanceOf(RestKiqrServerVerticle.class))); 52 | 53 | RestKiqrServerVerticle server = (RestKiqrServerVerticle) serverVerticle; 54 | assertThat(server.serverOptions.getPort(), is(equalTo(4711))); 55 | 56 | } 57 | 58 | @Test 59 | public void builderWithPort(){ 60 | RestKiqrServerVerticle.Builder builder = RestKiqrServerVerticle.Builder.serverBuilder(new KStreamBuilder(), new Properties()); 61 | 62 | AbstractVerticle serverVerticle = builder.withPort(4711).build(); 63 | assertThat(serverVerticle, is(instanceOf(RestKiqrServerVerticle.class))); 64 | 65 | RestKiqrServerVerticle server = (RestKiqrServerVerticle) serverVerticle; 66 | assertThat(server.serverOptions.getPort(), is(equalTo(4711))); 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/kv/AllKeyValuesQueryVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.kv; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.exceptions.SerdeNotFoundException; 22 | import org.apache.kafka.common.serialization.Serde; 23 | import org.apache.kafka.streams.KafkaStreams; 24 | import org.apache.kafka.streams.KeyValue; 25 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 26 | import org.apache.kafka.streams.state.KeyValueIterator; 27 | import org.apache.kafka.streams.state.QueryableStoreTypes; 28 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 29 | 30 | import java.util.*; 31 | 32 | /** 33 | * Created by ftr on 19/02/2017. 34 | */ 35 | public class AllKeyValuesQueryVerticle extends AbstractQueryVerticle { 36 | 37 | 38 | public AllKeyValuesQueryVerticle(String instanceId, KafkaStreams streams) { 39 | super(instanceId, streams); 40 | } 41 | 42 | 43 | @Override 44 | public void start() throws Exception { 45 | 46 | execute(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, (query, keySerde, valueSerde) -> { 47 | 48 | ReadOnlyKeyValueStore kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore()); 49 | MultiValuedKeyValueQueryResponse response; 50 | try (KeyValueIterator result = kvStore.all()) { 51 | if (result.hasNext()) { 52 | Map results = new HashMap<>(); 53 | while (result.hasNext()) { 54 | KeyValue kvEntry = result.next(); 55 | 56 | results.put(base64Encode(keySerde, kvEntry.key), base64Encode(valueSerde, kvEntry.value)); 57 | } 58 | return new MultiValuedKeyValueQueryResponse(results); 59 | } else { 60 | return new MultiValuedKeyValueQueryResponse(Collections.emptyMap()); 61 | } 62 | } 63 | }); 64 | 65 | } 66 | 67 | 68 | } 69 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/facade/KeyBasedQueryFacadeVerticle.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core.query.facade; 2 | 3 | import com.github.ftrossbach.kiqr.commons.config.Config; 4 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 5 | import com.github.ftrossbach.kiqr.core.ShareableStreamsMetadataProvider; 6 | import com.github.ftrossbach.kiqr.core.query.AbstractKiqrVerticle; 7 | import io.vertx.core.Future; 8 | import io.vertx.core.eventbus.ReplyException; 9 | import org.apache.kafka.common.serialization.Serde; 10 | import org.apache.kafka.streams.state.StreamsMetadata; 11 | 12 | /** 13 | * Created by ftr on 15/03/2017. 14 | */ 15 | public class KeyBasedQueryFacadeVerticle extends AbstractKiqrVerticle{ 16 | 17 | private final String listeningAddress; 18 | private final String queryAddressPrefix; 19 | 20 | public KeyBasedQueryFacadeVerticle(String listeningAddress, String queryAddressPrefix) { 21 | this.listeningAddress = listeningAddress; 22 | this.queryAddressPrefix = queryAddressPrefix; 23 | } 24 | 25 | @Override 26 | public void start(Future startFuture){ 27 | vertx.eventBus().consumer(listeningAddress, msg -> { 28 | REQ query = (REQ) msg.body(); 29 | 30 | ShareableStreamsMetadataProvider metadata = (ShareableStreamsMetadataProvider) vertx.sharedData().getLocalMap("metadata").get("metadata"); 31 | Serde serde = getSerde(query.getKeySerde()); 32 | 33 | Object deserializedKey = serde.deserializer().deserialize("?", ((HasKey)query).getKey()); 34 | 35 | StreamsMetadata streamsMetadata = metadata.metadataForKey(query.getStoreName(), deserializedKey, serde.serializer()); 36 | 37 | if(streamsMetadata == null ) { 38 | msg.fail(404, "Store not found: " + query.getStoreName()); 39 | } else if("unavailable".equals(streamsMetadata.host())){ 40 | 41 | msg.fail(503, "Streaming application currently unavailable"); 42 | } else { 43 | vertx.eventBus().send(queryAddressPrefix + streamsMetadata.host(), query, rep -> { 44 | 45 | if(rep.succeeded()){ 46 | RES queryResponse = (RES) rep.result().body(); 47 | 48 | msg.reply(queryResponse); 49 | } 50 | else { 51 | if(rep.cause() instanceof ReplyException){ 52 | ReplyException cause = (ReplyException) rep.cause(); 53 | msg.fail(cause.failureCode(), cause.getMessage()); 54 | } else { 55 | msg.fail(500, rep.cause().getMessage()); 56 | } 57 | } 58 | 59 | }); 60 | } 61 | 62 | 63 | }); 64 | 65 | startFuture.complete(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/SpecificBlockingRestKiqrClientImplTest.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.client.service.rest; 2 | 3 | import com.github.ftrossbach.kiqr.client.service.GenericBlockingKiqrClient; 4 | import org.apache.kafka.common.serialization.Serdes; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import org.mockito.Mock; 8 | import org.mockito.MockitoAnnotations; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.Optional; 13 | 14 | 15 | import static org.mockito.Mockito.*; 16 | import static org.junit.Assert.*; 17 | import static org.hamcrest.Matchers.*; 18 | 19 | /** 20 | * Created by ftr on 10/03/2017. 21 | */ 22 | public class SpecificBlockingRestKiqrClientImplTest { 23 | 24 | @Mock 25 | private GenericBlockingKiqrClient clientMock; 26 | 27 | private SpecificBlockingRestKiqrClientImpl unitUnderTest; 28 | 29 | @Before 30 | public void setUp() { 31 | MockitoAnnotations.initMocks(this); 32 | 33 | unitUnderTest = new SpecificBlockingRestKiqrClientImpl<>(clientMock, "store", String.class, Long.class, Serdes.String(), Serdes.Long()); 34 | 35 | } 36 | 37 | 38 | @Test 39 | public void scalar(){ 40 | 41 | when(clientMock.getScalarKeyValue(any(), any(), any(), any(), any(), any())).thenReturn(Optional.of(6L)); 42 | 43 | Optional result = unitUnderTest.getScalarKeyValue("key1"); 44 | 45 | assertTrue(result.isPresent()); 46 | assertThat(result.get(), is(equalTo(6L))); 47 | 48 | 49 | 50 | } 51 | 52 | @Test 53 | public void all(){ 54 | 55 | Map results = new HashMap<>(); 56 | results.put("key1", 6L); 57 | when(clientMock.getAllKeyValues(any(), eq(String.class), eq(Long.class), any(), any())).thenReturn(results); 58 | 59 | Map result = unitUnderTest.getAllKeyValues(); 60 | 61 | assertThat(result.size(), is(equalTo(1))); 62 | assertThat(result, hasEntry("key1", 6L)); 63 | 64 | } 65 | 66 | 67 | @Test 68 | public void range(){ 69 | 70 | Map results = new HashMap<>(); 71 | results.put("key1", 6L); 72 | when(clientMock.getRangeKeyValues(any(), eq(String.class), eq(Long.class), any(), any(), any(), any())).thenReturn(results); 73 | 74 | Map result = unitUnderTest.getRangeKeyValues("key1", "key2"); 75 | 76 | assertThat(result.size(), is(equalTo(1))); 77 | assertThat(result, hasEntry("key1", 6L)); 78 | 79 | } 80 | 81 | @Test 82 | public void window(){ 83 | 84 | Map results = new HashMap<>(); 85 | results.put(1L, 6L); 86 | when(clientMock.getWindow(anyString(), eq(String.class), eq("key1"), eq(Long.class), anyObject(), anyObject(), eq(0L), eq(1L))).thenReturn(results); 87 | 88 | Map result = unitUnderTest.getWindow("key1", 0L, 1L); 89 | 90 | assertThat(result.size(), is(equalTo(1))); 91 | assertThat(result, hasEntry(1L, 6L)); 92 | 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /rest-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | 23 | com.github.ftrossbach 24 | kiqr 25 | 0.0.2-SNAPSHOT 26 | 27 | 28 | kiqr-rest-client 29 | kiqr-rest-client 30 | REST client for KIQR 31 | https://github.com/ftrossbach/kiqr 32 | 33 | 34 | 35 | ${project.groupId} 36 | kiqr-commons 37 | 38 | 39 | org.apache.httpcomponents 40 | fluent-hc 41 | 4.5.3 42 | 43 | 44 | org.apache.kafka 45 | kafka-streams 46 | 47 | 48 | org.apache.kafka 49 | connect-json 50 | 51 | 52 | 53 | 54 | 55 | com.fasterxml.jackson.core 56 | jackson-databind 57 | 58 | 59 | com.github.ftrossbach 60 | kiqr-rest-server 61 | test 62 | 63 | 64 | io.vertx 65 | vertx-hazelcast 66 | test 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | org.apache.maven.plugins 75 | maven-failsafe-plugin 76 | 2.12.4 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /rest-client/src/main/java/com/github/ftrossbach/kiqr/client/service/rest/SpecificBlockingRestKiqrClientImpl.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.client.service.rest; 2 | 3 | import com.github.ftrossbach.kiqr.client.service.GenericBlockingKiqrClient; 4 | import com.github.ftrossbach.kiqr.client.service.SpecificBlockingKiqrClient; 5 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.Window; 6 | import org.apache.kafka.common.serialization.Serde; 7 | import java.util.Map; 8 | import java.util.Optional; 9 | 10 | /** 11 | * Created by ftr on 10/03/2017. 12 | */ 13 | public class SpecificBlockingRestKiqrClientImpl implements SpecificBlockingKiqrClient{ 14 | 15 | 16 | private final String store; 17 | private final Class keyClass; 18 | private final Class valueClass; 19 | private final Serde keySerde; 20 | private final Serde valueSerde; 21 | private final GenericBlockingKiqrClient genericClient; 22 | 23 | 24 | public SpecificBlockingRestKiqrClientImpl(String host, int port, String store, Class keyClass, Class valueClass, Serde keySerde, Serde valueSerde) { 25 | this.keyClass = keyClass; 26 | this.valueClass = valueClass; 27 | this.keySerde = keySerde; 28 | this.valueSerde = valueSerde; 29 | this.genericClient = initGenericService(host, port); 30 | this.store = store; 31 | } 32 | 33 | public SpecificBlockingRestKiqrClientImpl(GenericBlockingKiqrClient genericClient, String store, Class keyClass, Class valueClass, Serde keySerde, Serde valueSerde) { 34 | this.keyClass = keyClass; 35 | this.valueClass = valueClass; 36 | this.keySerde = keySerde; 37 | this.valueSerde = valueSerde; 38 | this.genericClient = genericClient; 39 | this.store = store; 40 | } 41 | 42 | protected GenericBlockingKiqrClient initGenericService(String host, int port){ 43 | return new GenericBlockingRestKiqrClientImpl(host, port); 44 | } 45 | 46 | 47 | @Override 48 | public Optional getScalarKeyValue( K key) { 49 | return genericClient.getScalarKeyValue(store, keyClass, key, valueClass, keySerde, valueSerde); 50 | } 51 | 52 | @Override 53 | public Map getAllKeyValues() { 54 | return genericClient.getAllKeyValues(store, keyClass, valueClass, keySerde, valueSerde); 55 | } 56 | 57 | @Override 58 | public Map getRangeKeyValues( K from, K to) { 59 | return genericClient.getRangeKeyValues(store, keyClass, valueClass, keySerde, valueSerde, from, to); 60 | } 61 | 62 | @Override 63 | public Map getWindow( K key, long from, long to) { 64 | return genericClient.getWindow(store, keyClass, key, valueClass, keySerde, valueSerde, from, to); 65 | } 66 | 67 | @Override 68 | public Optional count(String store) { 69 | return genericClient.count(store); 70 | } 71 | 72 | @Override 73 | public Map getSession(K key) { 74 | return genericClient.getSession(store, keyClass, key, valueClass, keySerde, valueSerde); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/CountQueryHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.RuntimeVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.http.HttpServerOptions; 23 | import io.vertx.ext.unit.Async; 24 | import io.vertx.ext.unit.TestContext; 25 | import io.vertx.ext.unit.junit.RunTestOnContext; 26 | import io.vertx.ext.unit.junit.VertxUnitRunner; 27 | import org.apache.kafka.common.serialization.Serdes; 28 | import org.junit.Before; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | import org.junit.runner.RunWith; 32 | import java.util.Collections; 33 | import java.util.HashMap; 34 | import java.util.Map; 35 | 36 | 37 | import static org.mockito.Mockito.mock; 38 | 39 | /** 40 | * Created by ftr on 06/03/2017. 41 | */ 42 | @RunWith(VertxUnitRunner.class) 43 | public class CountQueryHttpServerTest { 44 | 45 | @Rule 46 | public final RunTestOnContext rule = new RunTestOnContext(); 47 | 48 | 49 | 50 | @Before 51 | public void setUp(TestContext context) throws Exception{ 52 | rule.vertx().eventBus().registerDefaultCodec(StoreWideQuery.class, new KiqrCodec(StoreWideQuery.class)); 53 | rule.vertx().eventBus().registerDefaultCodec(KeyValueStoreCountQuery.class, new KiqrCodec<>(KeyValueStoreCountQuery.class)); 54 | 55 | 56 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummySuccessfulVerticle()), context.asyncAssertSuccess()); 57 | } 58 | 59 | 60 | @Test 61 | public void readCount(TestContext context) throws Exception { 62 | 63 | Async async = context.async(); 64 | 65 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 66 | 67 | 68 | 69 | rule.vertx().eventBus().consumer(Config.COUNT_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 70 | context.assertTrue(msg.body() instanceof HasStoreName); 71 | HasStoreName query = (HasStoreName) msg.body(); 72 | 73 | context.assertEquals("store", query.getStoreName()); 74 | 75 | 76 | msg.reply(42L); 77 | 78 | }); 79 | 80 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/count"), res ->{ 81 | 82 | context.assertEquals(200, res.statusCode()); 83 | async.complete(); 84 | }).end(); 85 | } 86 | 87 | 88 | 89 | 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/kv/RangeKeyValueQueryVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.kv; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.MultiValuedKeyValueQueryResponse; 20 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.RangeKeyValueQuery; 21 | import com.github.ftrossbach.kiqr.core.query.AbstractQueryVerticle; 22 | import com.github.ftrossbach.kiqr.core.query.exceptions.SerdeNotFoundException; 23 | import org.apache.kafka.common.serialization.Serde; 24 | import org.apache.kafka.streams.KafkaStreams; 25 | import org.apache.kafka.streams.KeyValue; 26 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 27 | import org.apache.kafka.streams.state.KeyValueIterator; 28 | import org.apache.kafka.streams.state.QueryableStoreTypes; 29 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 30 | 31 | import java.util.Collections; 32 | import java.util.HashMap; 33 | import java.util.Map; 34 | 35 | /** 36 | * Created by ftr on 19/02/2017. 37 | */ 38 | public class RangeKeyValueQueryVerticle extends AbstractQueryVerticle { 39 | 40 | 41 | public RangeKeyValueQueryVerticle(String instanceId, KafkaStreams streams) { 42 | super(instanceId, streams); 43 | } 44 | 45 | 46 | @Override 47 | public void start() throws Exception { 48 | 49 | execute(Config.RANGE_KEY_VALUE_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> { 50 | 51 | RangeKeyValueQuery query = (RangeKeyValueQuery) abstractQuery; 52 | ReadOnlyKeyValueStore kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore()); 53 | MultiValuedKeyValueQueryResponse response; 54 | try (KeyValueIterator result = kvStore.range(deserializeObject(keySerde, query.getFrom()), deserializeObject(keySerde, query.getTo()))) { 55 | if (result.hasNext()) { 56 | Map results = new HashMap<>(); 57 | while (result.hasNext()) { 58 | KeyValue kvEntry = result.next(); 59 | results.put(base64Encode(keySerde, kvEntry.key), base64Encode(valueSerde, kvEntry.value)); 60 | } 61 | return new MultiValuedKeyValueQueryResponse(results); 62 | } else { 63 | return new MultiValuedKeyValueQueryResponse(Collections.emptyMap()); 64 | } 65 | } 66 | } ); 67 | 68 | 69 | } 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/AbstractQueryVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.AbstractQuery; 19 | import com.github.ftrossbach.kiqr.core.query.exceptions.ScalarValueNotFoundException; 20 | import com.github.ftrossbach.kiqr.core.query.exceptions.SerdeNotFoundException; 21 | import io.vertx.core.logging.Logger; 22 | import io.vertx.core.logging.LoggerFactory; 23 | import org.apache.kafka.common.serialization.Serde; 24 | import org.apache.kafka.streams.KafkaStreams; 25 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 26 | 27 | /** 28 | * Created by ftr on 05/03/2017. 29 | */ 30 | public class AbstractQueryVerticle extends AbstractKiqrVerticle{ 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(AbstractQueryVerticle.class); 33 | protected final String instanceId; 34 | protected final KafkaStreams streams; 35 | 36 | public AbstractQueryVerticle(String instanceId, KafkaStreams streams) { 37 | this.streams = streams; 38 | this.instanceId = instanceId; 39 | } 40 | 41 | 42 | 43 | protected void execute(String addressPrefix, MappingFunction mapper){ 44 | vertx.eventBus().consumer(addressPrefix + instanceId, msg -> { 45 | AbstractQuery query = null; 46 | try { 47 | query = (AbstractQuery) msg.body(); 48 | 49 | Serde keySerde = getSerde(query.getKeySerde()); 50 | Serde valueSerde = getSerde(query.getValueSerde()); 51 | 52 | Object response = mapper.apply(query, keySerde, valueSerde); 53 | msg.reply(response); 54 | } catch (ScalarValueNotFoundException e){ 55 | msg.fail(404, e.getMessage()); 56 | } catch (SerdeNotFoundException e) { 57 | LOG.error("Serde not instantiable", e); 58 | msg.fail(400, e.getMessage()); 59 | } catch (InvalidStateStoreException e) { 60 | LOG.error(String.format("Store %s not queriable. Could be due to wrong store type or rebalancing", query.getStoreName())); 61 | msg.fail(500, String.format("Store not available due to rebalancing or wrong store type")); 62 | } catch (RuntimeException e) { 63 | LOG.error("Unexpected exception", e); 64 | msg.fail(500, e.getMessage()); 65 | } 66 | }); 67 | } 68 | 69 | @FunctionalInterface 70 | protected interface MappingFunction{ 71 | 72 | Object apply(AbstractQuery query, Serde keySerde, Serde valueSerde); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/SessionWindowQueryHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.RuntimeVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.http.HttpServerOptions; 23 | import io.vertx.ext.unit.Async; 24 | import io.vertx.ext.unit.TestContext; 25 | import io.vertx.ext.unit.junit.RunTestOnContext; 26 | import io.vertx.ext.unit.junit.VertxUnitRunner; 27 | import org.apache.kafka.common.serialization.Serdes; 28 | import org.junit.Before; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | import org.junit.runner.RunWith; 32 | import java.util.*; 33 | 34 | 35 | import static org.mockito.Mockito.mock; 36 | 37 | /** 38 | * Created by ftr on 06/03/2017. 39 | */ 40 | @RunWith(VertxUnitRunner.class) 41 | public class SessionWindowQueryHttpServerTest { 42 | 43 | @Rule 44 | public final RunTestOnContext rule = new RunTestOnContext(); 45 | 46 | 47 | 48 | @Before 49 | public void setUp(TestContext context) throws Exception{ 50 | rule.vertx().eventBus().registerDefaultCodec(KeyBasedQuery.class, new KiqrCodec(KeyBasedQuery.class)); 51 | rule.vertx().eventBus().registerDefaultCodec(SessionQueryResponse.class, new KiqrCodec<>(SessionQueryResponse.class)); 52 | 53 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummySuccessfulVerticle()), context.asyncAssertSuccess()); 54 | } 55 | 56 | 57 | @Test 58 | public void readSessionWindowValue(TestContext context) throws Exception { 59 | 60 | Async async = context.async(); 61 | 62 | 63 | 64 | rule.vertx().eventBus().consumer(Config.SESSION_QUERY_FACADE_ADDRESS, msg -> { 65 | context.assertTrue(msg.body() instanceof KeyBasedQuery); 66 | KeyBasedQuery query = (KeyBasedQuery) msg.body(); 67 | 68 | context.assertEquals("store", query.getStoreName()); 69 | context.assertEquals(Serdes.String().getClass().getName(), query.getKeySerde()); 70 | context.assertEquals(Serdes.Long().getClass().getName(), query.getValueSerde()); 71 | context.assertTrue(query.getKey().length > 0); 72 | 73 | List results = new ArrayList<>(); 74 | results.add(new Window(0,100, "hurz")); 75 | results.add(new Window(101, 200, "burz")); 76 | 77 | msg.reply(new SessionQueryResponse(results)); 78 | 79 | }); 80 | 81 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/session/store/key?keySerde=%s&valueSerde=%s&from=1&to=2", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 82 | 83 | context.assertEquals(200, res.statusCode()); 84 | async.complete(); 85 | }).end(); 86 | } 87 | 88 | 89 | 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /examples/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | 23 | com.github.ftrossbach 24 | kiqr 25 | 0.0.2-SNAPSHOT 26 | 27 | 28 | kiqr-examples 29 | 30 | 31 | 32 | com.github.ftrossbach 33 | kiqr-core 34 | 35 | 36 | com.github.ftrossbach 37 | kiqr-rest-client 38 | 39 | 40 | com.github.ftrossbach 41 | kiqr-rest-server 42 | 43 | 44 | io.vertx 45 | vertx-core 46 | 47 | 48 | io.vertx 49 | vertx-hazelcast 50 | 51 | 52 | org.apache.kafka 53 | kafka-streams 54 | 55 | 56 | 57 | 58 | 59 | 60 | org.apache.maven.plugins 61 | maven-shade-plugin 62 | 3.0.0 63 | 64 | 65 | package 66 | 67 | shade 68 | 69 | 70 | 71 | 72 | 73 | io.vertx.core.Launcher 74 | com.github.ftrossbach.kiqr.examples.MainVerticle 75 | 76 | 77 | 78 | 79 | ${project.build.directory}/${project.artifactId}-${project.version}-fat.jar 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /core/src/main/java/com/github/ftrossbach/kiqr/core/query/facade/ScatterGatherQueryFacadeVerticle.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | *

4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | *

8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | *

10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.facade; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 19 | import com.github.ftrossbach.kiqr.core.ShareableStreamsMetadataProvider; 20 | import io.vertx.core.AbstractVerticle; 21 | import io.vertx.core.CompositeFuture; 22 | import io.vertx.core.Future; 23 | import io.vertx.core.eventbus.Message; 24 | import io.vertx.core.eventbus.ReplyException; 25 | import org.apache.kafka.streams.state.StreamsMetadata; 26 | 27 | import java.util.Collection; 28 | import java.util.List; 29 | import java.util.function.BinaryOperator; 30 | import java.util.function.Supplier; 31 | import java.util.stream.Collectors; 32 | 33 | /** 34 | * Created by ftr on 22/02/2017. 35 | */ 36 | public class ScatterGatherQueryFacadeVerticle extends AbstractVerticle { 37 | 38 | private final String listeningAddress; 39 | private final String queryAddressPrefix; 40 | private final Supplier identity; 41 | private final BinaryOperator reducer; 42 | 43 | public ScatterGatherQueryFacadeVerticle(String listeningAddress, String queryAddressPrefix, Supplier identity, BinaryOperator reducer) { 44 | this.listeningAddress = listeningAddress; 45 | this.queryAddressPrefix = queryAddressPrefix; 46 | this.identity = identity; 47 | this.reducer = reducer; 48 | } 49 | 50 | @Override 51 | public void start() throws Exception { 52 | vertx.eventBus().consumer(listeningAddress, msg -> { 53 | HasStoreName query = (HasStoreName) msg.body(); 54 | 55 | 56 | ShareableStreamsMetadataProvider metadataProvider = (ShareableStreamsMetadataProvider) vertx.sharedData().getLocalMap("metadata").get("metadata"); 57 | 58 | Collection allMetadata = metadataProvider.allMetadataForStore(query.getStoreName()); 59 | 60 | 61 | if (allMetadata.isEmpty()) { 62 | msg.fail(404, "No instance for store found: " + query.getStoreName()); 63 | } else if (allMetadata.stream().anyMatch(meta -> "unavailable".equals(meta.host()))) { 64 | msg.fail(503, "Streaming application currently unavailable"); 65 | } 66 | 67 | 68 | List results = allMetadata.stream().map(StreamsMetadata::host) 69 | .map(instanceId -> { 70 | Future> future = Future.>future(); 71 | vertx.eventBus().send(queryAddressPrefix + instanceId, query, future.completer()); 72 | return future; 73 | }).collect(Collectors.toList()); 74 | 75 | CompositeFuture all = CompositeFuture.all(results); 76 | 77 | 78 | all.setHandler(compoundFutureHandler -> { 79 | 80 | if (compoundFutureHandler.succeeded()) { 81 | List> list = compoundFutureHandler.result().list(); 82 | 83 | RES reduced = list.stream().map(message -> message.body()).reduce(identity.get(), reducer); 84 | msg.reply(reduced); 85 | } else { 86 | ReplyException cause = (ReplyException) compoundFutureHandler.cause(); 87 | msg.fail(cause.failureCode(), cause.getMessage()); 88 | } 89 | }); 90 | 91 | 92 | }); 93 | 94 | } 95 | } 96 | 97 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/CountQueryBlockingRestKiqrServiceImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.ConnectionException; 19 | import com.github.ftrossbach.kiqr.client.service.QueryExecutionException; 20 | import com.github.ftrossbach.kiqr.commons.config.Config; 21 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyBasedQuery; 22 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyValueStoreCountQuery; 23 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.ScalarKeyValueQueryResponse; 24 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.StoreWideQuery; 25 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 26 | import io.vertx.core.http.HttpServerOptions; 27 | import io.vertx.ext.unit.Async; 28 | import io.vertx.ext.unit.TestContext; 29 | import io.vertx.ext.unit.junit.RunTestOnContext; 30 | import io.vertx.ext.unit.junit.VertxUnitRunner; 31 | import org.apache.kafka.common.serialization.Serdes; 32 | import org.junit.Before; 33 | import org.junit.Rule; 34 | import org.junit.Test; 35 | import org.junit.runner.RunWith; 36 | import java.util.Base64; 37 | import java.util.Optional; 38 | 39 | /** 40 | * Created by ftr on 07/03/2017. 41 | */ 42 | @RunWith(VertxUnitRunner.class) 43 | public class CountQueryBlockingRestKiqrServiceImplTest { 44 | 45 | GenericBlockingRestKiqrClientImpl unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", 4567); 46 | 47 | @Rule 48 | public final RunTestOnContext rule = new RunTestOnContext(); 49 | 50 | 51 | @Before 52 | public void setUp(){ 53 | rule.vertx().eventBus().registerDefaultCodec(KeyValueStoreCountQuery.class, new KiqrCodec(KeyValueStoreCountQuery.class)); 54 | 55 | 56 | } 57 | 58 | 59 | @Test 60 | public void successfulQuery(TestContext context){ 61 | Async async = context.async(); 62 | 63 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 64 | 65 | rule.vertx().eventBus().consumer(Config.COUNT_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 66 | 67 | msg.reply(42L); 68 | 69 | }); 70 | 71 | rule.vertx().executeBlocking(future -> { 72 | 73 | Optional scalarKeyValue = unitUnderTest.count("store"); 74 | context.assertTrue(scalarKeyValue.isPresent()); 75 | context.assertEquals(42L, scalarKeyValue.get()); 76 | async.complete(); 77 | 78 | future.complete(); 79 | }, context.asyncAssertSuccess()); 80 | 81 | })); 82 | 83 | } 84 | 85 | @Test 86 | public void notFound(TestContext context){ 87 | Async async = context.async(); 88 | 89 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 90 | 91 | rule.vertx().eventBus().consumer(Config.COUNT_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 92 | 93 | msg.fail(404, "does not matter"); 94 | 95 | }); 96 | 97 | rule.vertx().executeBlocking(future -> { 98 | 99 | Optional scalarKeyValue = unitUnderTest.count("idontexist"); 100 | context.assertFalse(scalarKeyValue.isPresent()); 101 | async.complete(); 102 | 103 | future.complete(); 104 | }, context.asyncAssertSuccess()); 105 | 106 | })); 107 | 108 | } 109 | 110 | 111 | 112 | @Test 113 | public void internalServerError(TestContext context){ 114 | Async async = context.async(); 115 | 116 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 117 | 118 | rule.vertx().eventBus().consumer(Config.COUNT_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 119 | 120 | msg.fail(500, "does not matter"); 121 | 122 | }); 123 | 124 | rule.vertx().executeBlocking(future -> { 125 | 126 | 127 | Optional scalarKeyValue = unitUnderTest.count("store"); 128 | context.fail(); 129 | 130 | 131 | 132 | }, context.asyncAssertFailure(ex -> { 133 | context.assertTrue(ex instanceof QueryExecutionException); 134 | async.complete(); 135 | })); 136 | 137 | })); 138 | 139 | } 140 | 141 | 142 | } 143 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/kv/KeyValueCountVerticleTest.java: -------------------------------------------------------------------------------- 1 | package com.github.ftrossbach.kiqr.core.query.kv; 2 | 3 | import com.github.ftrossbach.kiqr.commons.config.Config; 4 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyValueStoreCountQuery; 5 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.MultiValuedKeyValueQueryResponse; 6 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.RangeKeyValueQuery; 7 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 8 | import com.github.ftrossbach.kiqr.core.query.SimpleKeyValueIterator; 9 | import io.vertx.core.eventbus.ReplyException; 10 | import io.vertx.ext.unit.TestContext; 11 | import io.vertx.ext.unit.junit.RunTestOnContext; 12 | import io.vertx.ext.unit.junit.VertxUnitRunner; 13 | import org.apache.kafka.common.serialization.Serdes; 14 | import org.apache.kafka.streams.KafkaStreams; 15 | import org.apache.kafka.streams.KeyValue; 16 | import org.apache.kafka.streams.errors.InvalidStateStoreException; 17 | import org.apache.kafka.streams.state.KeyValueIterator; 18 | import org.apache.kafka.streams.state.QueryableStoreType; 19 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 20 | import org.junit.Before; 21 | import org.junit.Rule; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | 25 | 26 | import static org.mockito.Matchers.any; 27 | import static org.mockito.Matchers.eq; 28 | import static org.mockito.Mockito.mock; 29 | import static org.mockito.Mockito.when; 30 | 31 | /** 32 | * Created by ftr on 17/03/2017. 33 | */ 34 | @RunWith(VertxUnitRunner.class) 35 | public class KeyValueCountVerticleTest { 36 | 37 | @Rule 38 | public RunTestOnContext rule = new RunTestOnContext(); 39 | 40 | @Before 41 | public void setUp(){ 42 | rule.vertx().eventBus().registerDefaultCodec(KeyValueStoreCountQuery.class, new KiqrCodec(KeyValueStoreCountQuery.class)); 43 | } 44 | 45 | @Test 46 | public void success(TestContext context){ 47 | KafkaStreams streamMock = mock(KafkaStreams.class); 48 | ReadOnlyKeyValueStore storeMock = mock(ReadOnlyKeyValueStore.class); 49 | KeyValueIterator iteratorMock = mock(KeyValueIterator.class); 50 | 51 | when(streamMock.store(eq("store"), any(QueryableStoreType.class))).thenReturn(storeMock); 52 | when(storeMock.approximateNumEntries()).thenReturn(42L); 53 | 54 | 55 | rule.vertx().deployVerticle(new KeyValueCountVerticle("host", streamMock), context.asyncAssertSuccess(deployment->{ 56 | 57 | KeyValueStoreCountQuery query = new KeyValueStoreCountQuery("store"); 58 | 59 | rule.vertx().eventBus().send(Config.COUNT_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host", query, context.asyncAssertSuccess(reply ->{ 60 | 61 | context.assertTrue(reply.body() instanceof Long); 62 | context.assertEquals(42L, reply.body()); 63 | 64 | })); 65 | 66 | 67 | })); 68 | 69 | 70 | } 71 | 72 | 73 | 74 | 75 | 76 | @Test 77 | public void illegalStateStoreExceptionOnQuery(TestContext context){ 78 | KafkaStreams streamMock = mock(KafkaStreams.class); 79 | ReadOnlyKeyValueStore storeMock = mock(ReadOnlyKeyValueStore.class); 80 | KeyValueIterator iteratorMock = mock(KeyValueIterator.class); 81 | 82 | when(streamMock.store(eq("store"), any(QueryableStoreType.class))).thenReturn(storeMock); 83 | when(storeMock.approximateNumEntries()).thenThrow(InvalidStateStoreException.class); 84 | 85 | rule.vertx().deployVerticle(new KeyValueCountVerticle("host", streamMock), context.asyncAssertSuccess(deployment->{ 86 | 87 | KeyValueStoreCountQuery query = new KeyValueStoreCountQuery("store"); 88 | 89 | rule.vertx().eventBus().send(Config.COUNT_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host", query, context.asyncAssertFailure(handler ->{ 90 | 91 | context.assertTrue(handler instanceof ReplyException); 92 | ReplyException ex = (ReplyException) handler; 93 | context.assertEquals(500, ex.failureCode()); 94 | 95 | })); 96 | 97 | })); 98 | 99 | } 100 | 101 | @Test 102 | public void unexpectedExceptionOnQuery(TestContext context){ 103 | KafkaStreams streamMock = mock(KafkaStreams.class); 104 | ReadOnlyKeyValueStore storeMock = mock(ReadOnlyKeyValueStore.class); 105 | KeyValueIterator iteratorMock = mock(KeyValueIterator.class); 106 | 107 | when(streamMock.store(eq("store"), any(QueryableStoreType.class))).thenReturn(storeMock); 108 | when(storeMock.approximateNumEntries()).thenThrow(IllegalArgumentException.class); 109 | 110 | rule.vertx().deployVerticle(new KeyValueCountVerticle("host", streamMock), context.asyncAssertSuccess(deployment->{ 111 | 112 | KeyValueStoreCountQuery query = new KeyValueStoreCountQuery("store"); 113 | 114 | rule.vertx().eventBus().send(Config.COUNT_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host", query, context.asyncAssertFailure(handler ->{ 115 | 116 | context.assertTrue(handler instanceof ReplyException); 117 | ReplyException ex = (ReplyException) handler; 118 | context.assertEquals(500, ex.failureCode()); 119 | 120 | })); 121 | 122 | })); 123 | 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/ScalarQueryHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyBasedQuery; 20 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.ScalarKeyValueQueryResponse; 21 | import com.github.ftrossbach.kiqr.core.RuntimeVerticle; 22 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 23 | import io.vertx.core.http.HttpServerOptions; 24 | import io.vertx.ext.unit.Async; 25 | import io.vertx.ext.unit.TestContext; 26 | import io.vertx.ext.unit.junit.RunTestOnContext; 27 | import io.vertx.ext.unit.junit.VertxUnitRunner; 28 | import org.apache.kafka.common.serialization.Serdes; 29 | import org.junit.Before; 30 | import org.junit.Rule; 31 | import org.junit.Test; 32 | import org.junit.runner.RunWith; 33 | import static org.mockito.Mockito.*; 34 | 35 | /** 36 | * Created by ftr on 06/03/2017. 37 | */ 38 | @RunWith(VertxUnitRunner.class) 39 | public class ScalarQueryHttpServerTest { 40 | 41 | @Rule 42 | public final RunTestOnContext rule = new RunTestOnContext(); 43 | 44 | 45 | 46 | @Before 47 | public void setUp(TestContext context) throws Exception{ 48 | rule.vertx().eventBus().registerDefaultCodec(KeyBasedQuery.class, new KiqrCodec(KeyBasedQuery.class)); 49 | rule.vertx().eventBus().registerDefaultCodec(ScalarKeyValueQueryResponse.class, new KiqrCodec(ScalarKeyValueQueryResponse.class)); 50 | 51 | 52 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummySuccessfulVerticle()), context.asyncAssertSuccess()); 53 | } 54 | 55 | 56 | @Test 57 | public void readScalarValue(TestContext context) throws Exception { 58 | 59 | Async async = context.async(); 60 | 61 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 62 | 63 | 64 | 65 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 66 | context.assertTrue(msg.body() instanceof KeyBasedQuery); 67 | KeyBasedQuery query = (KeyBasedQuery) msg.body(); 68 | 69 | context.assertEquals("store", query.getStoreName()); 70 | context.assertEquals(Serdes.String().getClass().getName(), query.getKeySerde()); 71 | context.assertEquals(Serdes.Long().getClass().getName(), query.getValueSerde()); 72 | context.assertTrue(query.getKey().length > 0); 73 | 74 | msg.reply(new ScalarKeyValueQueryResponse("test")); 75 | 76 | }); 77 | 78 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/values/key?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 79 | 80 | context.assertEquals(200, res.statusCode()); 81 | async.complete(); 82 | }).end(); 83 | } 84 | 85 | @Test 86 | public void scalarValueNotFound(TestContext context) { 87 | 88 | Async async = context.async(); 89 | 90 | 91 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 92 | 93 | msg.fail(404, "irrelevant"); 94 | 95 | }); 96 | 97 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/values/key?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 98 | 99 | context.assertEquals(404, res.statusCode()); 100 | async.complete(); 101 | }).end(); 102 | } 103 | 104 | @Test 105 | public void scalarValueNoKeySerde(TestContext context) { 106 | 107 | Async async = context.async(); 108 | 109 | 110 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/values/key?valueSerde=%s", Serdes.Long().getClass().getName()), res ->{ 111 | 112 | context.assertEquals(400, res.statusCode()); 113 | async.complete(); 114 | }).end(); 115 | } 116 | 117 | @Test 118 | public void scalarValueNoValueSerde(TestContext context) { 119 | 120 | Async async = context.async(); 121 | 122 | 123 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/values/key?keySerde=%s", Serdes.Long().getClass().getName()), res ->{ 124 | 125 | context.assertEquals(400, res.statusCode()); 126 | async.complete(); 127 | }).end(); 128 | } 129 | 130 | @Test 131 | public void scalarInternalServerError(TestContext context){ 132 | Async async = context.async(); 133 | 134 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 135 | 136 | 137 | 138 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 139 | msg.fail(500, "msg"); 140 | 141 | }); 142 | 143 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store/values/key?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 144 | 145 | context.assertEquals(500, res.statusCode()); 146 | async.complete(); 147 | }).end(); 148 | } 149 | 150 | 151 | 152 | 153 | } 154 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/facade/KeyValueQueryFacadeVerticleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.facade; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.ShareableStreamsMetadataProvider; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.eventbus.ReplyException; 23 | import io.vertx.ext.unit.TestContext; 24 | import io.vertx.ext.unit.junit.RunTestOnContext; 25 | import io.vertx.ext.unit.junit.VertxUnitRunner; 26 | import org.apache.kafka.common.serialization.Serdes; 27 | import org.apache.kafka.common.serialization.Serializer; 28 | import org.apache.kafka.streams.state.HostInfo; 29 | import org.apache.kafka.streams.state.StreamsMetadata; 30 | import org.junit.Before; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import java.util.Collections; 35 | 36 | 37 | import static org.mockito.Matchers.any; 38 | import static org.mockito.Matchers.anyString; 39 | import static org.mockito.Mockito.mock; 40 | import static org.mockito.Mockito.when; 41 | 42 | /** 43 | * Created by ftr on 05/03/2017. 44 | */ 45 | @RunWith(VertxUnitRunner.class) 46 | public class KeyValueQueryFacadeVerticleTest { 47 | @Rule 48 | public RunTestOnContext rule = new RunTestOnContext(); 49 | 50 | @Before 51 | public void setUp(){ 52 | rule.vertx().eventBus().registerDefaultCodec(KeyBasedQuery.class, new KiqrCodec(KeyBasedQuery.class)); 53 | rule.vertx().eventBus().registerDefaultCodec(ScalarKeyValueQueryResponse.class, new KiqrCodec(ScalarKeyValueQueryResponse.class)); 54 | 55 | } 56 | 57 | @Test 58 | public void success(TestContext context){ 59 | 60 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 61 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 62 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(host); 63 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 64 | 65 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_ADDRESS_PREFIX + "host", msg -> { 66 | msg.reply(new ScalarKeyValueQueryResponse("value")); 67 | }); 68 | 69 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, Config.KEY_VALUE_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 70 | 71 | KeyBasedQuery query = new KeyBasedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(), Serdes.String().getClass().getName()); 72 | 73 | rule.vertx().eventBus().send(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertSuccess(reply ->{ 74 | 75 | context.assertTrue(reply.body() instanceof ScalarKeyValueQueryResponse); 76 | ScalarKeyValueQueryResponse response = (ScalarKeyValueQueryResponse) reply.body(); 77 | context.assertEquals("value", response.getValue()); 78 | 79 | })); 80 | 81 | 82 | })); 83 | 84 | } 85 | 86 | @Test 87 | public void forwardingFailureDuringQuery(TestContext context){ 88 | 89 | 90 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 91 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 92 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(host); 93 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 94 | 95 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_ADDRESS_PREFIX + "host", msg -> { 96 | msg.fail(400, "msg"); 97 | }); 98 | 99 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, Config.KEY_VALUE_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 100 | 101 | KeyBasedQuery query = new KeyBasedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(), Serdes.String().getClass().getName()); 102 | 103 | rule.vertx().eventBus().send(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 104 | 105 | context.assertTrue(handler instanceof ReplyException); 106 | ReplyException ex = (ReplyException) handler; 107 | context.assertEquals(400, ex.failureCode()); 108 | context.assertEquals("msg", ex.getMessage()); 109 | 110 | })); 111 | })); 112 | 113 | } 114 | 115 | @Test 116 | public void forwardingFailureDuringInstanceLookup(TestContext context){ 117 | 118 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 119 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 120 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(null); 121 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 122 | 123 | 124 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, Config.KEY_VALUE_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 125 | 126 | KeyBasedQuery query = new KeyBasedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(), Serdes.String().getClass().getName()); 127 | 128 | rule.vertx().eventBus().send(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 129 | 130 | context.assertTrue(handler instanceof ReplyException); 131 | ReplyException ex = (ReplyException) handler; 132 | context.assertEquals(404, ex.failureCode()); 133 | 134 | })); 135 | })); 136 | 137 | } 138 | 139 | } 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ftrossbach/kiqr.svg?branch=master)](https://travis-ci.org/ftrossbach/kiqr) 2 | [![Coverage Status](https://coveralls.io/repos/github/ftrossbach/kiqr/badge.svg)](https://coveralls.io/github/ftrossbach/kiqr) 3 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | 5 | 6 | 7 | #KIQR - Kafka Interactive Query Runtime 8 | 9 | This project aims at providing a general purpose runtime for interactive queries. 10 | It uses Vert.x for coordination between cluster instances. 11 | 12 | ## Why 13 | Apache Kafka has a cool feature called "Interactive Queries" that enables you to query the internal state of a 14 | Kafka Streams application. That's pretty cool, but if you run your streams application in a distributed manner where 15 | different instances of your app are assigned different partitions of your inbound Kafka topics, each 16 | instance is only aware of the messages that come it's way. If you want a reliable query environment, you need to build 17 | a layer that is aware of those instances and which instance is responsible for which key. 18 | 19 | KIQR was started as a vehicle to get deeper into the Interactive Query feature and to pick up some Vert.x knowledge 20 | along the way. It probably will never get past that point, but I'll be happy if it is of any use to someone else. 21 | 22 | ## How 23 | The property "application.server" in KafkaStreams lets each instance share information of its coordinates on the 24 | network in host:port format with all other instances via Kafka protocol mechanisms. KIQR uses this feature, but in 25 | a different way. On startup, instances assign themselves a UUID as host (and a irrelevant value for port). It then 26 | uses this UUID to register at Vert.x' event bus. If you run Vert.x in cluster mode, this will be a distributed event 27 | bus, meaning that the instances can talk to each other on that bus. So any instance can query one of KafkaStreams' 28 | metadata methods and know at which address on the event bus to direct the query at. You can use any of Vert.x' supported 29 | cluster mechanisms. 30 | 31 | For serialization purposes, Kafka's Serdes (Serializer/Deserializers) are used as they are required to interact with 32 | Kafka anyway. They need to be on the classpath of both client and server. 33 | 34 | ## Client 35 | At the moment, KIQR allows queries via HTTP. There is a server and a client module. More clients are certainly imaginable. 36 | 37 | ## Examples 38 | 39 | 40 | ### Server Runtime 41 | Running a streams application in the KIQR runtime 42 | ``` 43 | Properties streamProps = new Properties(); 44 | streamProps.put(StreamsConfig.APPLICATION_ID_CONFIG, UUID.randomUUID().toString()); 45 | streamProps.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "my-streaming-app"); 46 | streamProps.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); 47 | streamProps.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 48 | streamProps.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName()); 49 | 50 | KStreamBuilder builder = new KStreamBuilder(); 51 | KTable kv = builder.table(Serdes.String(), Serdes.Long(), TOPIC, "kv"); 52 | kv.toStream().groupByKey().count(TimeWindows.of(10000L), "window"); 53 | 54 | Vertx vertx = ...;// get your vertx from somewhere 55 | 56 | RestKiqrServerVerticle.Builder verticleBuilder = RestKiqrServerVerticle.Builder.serverBuilder(builder, streamProps).withPort(PORT2); 57 | 58 | vertx.deploy(verticleBuilder.build()); 59 | 60 | ``` 61 | 62 | There are multiple ways to get hold of the Vertx object. 63 | If you run in a clustered environment, check out the [Vert.x docs on clustering](http://vertx.io/docs/#clustering). 64 | For single instance tests, it can be as easy as calling 65 | ``` 66 | Vertx vertx = Vertx.vertx(); 67 | ``` 68 | 69 | ### Client 70 | There is a rest client that does all the deserialization for you, so you only interact with the actual data types and 71 | not some serialized byte arrays. The client is written plain Java without Vert.x. 72 | It only depends on Kafka Streams (for the serdes), Apache's HTTP client and Jackson. 73 | 74 | There is a generic variant were you specify the class and serde of both key and value type on each call, and a specific 75 | one that can only be used for one key-value combination but gets these set in its constructor. 76 | 77 | 78 | ``` 79 | GenericBlockingKiqrClient client = new GenericBlockingRestKiqrClientImpl("localhost", port); 80 | 81 | //querying key "key1" from key-value store "kv" with String keys and Long values 82 | Optional result = client.getScalarKeyValue("kv", String.class, "key1", Long.class, Serdes.String(), Serdes.Long()); 83 | 84 | //querying count of entries from key-value store "kv" 85 | Optional result = client.count("kv"); 86 | 87 | //querying all keys from store "kv" with String keys and Long values 88 | Map result = client.getAllKeyValues("kv", String.class, Long.class, Serdes.String(), Serdes.Long()); 89 | 90 | //querying key range "key1" to "key3" from store "kv" with String keys and Long values 91 | Map result = client.getRangeKeyValues("kv", String.class, Long.class, Serdes.String(), Serdes.Long(), "key1", "key3"); 92 | 93 | //querying windows for "key1" from epoch time 1 to epoch time 1000 from store "window" with String keys and Long values 94 | Map result = client.getWindow("window", String.class, "key1", Long.class, Serdes.String(), Serdes.Long(), 1L, 1000L); 95 | ``` 96 | 97 | Those methods look a bit clunky, that's why there is a specific variant: 98 | 99 | ``` 100 | //constructing a client for a store called "kv" with String keys and long values 101 | SpecificBlockingKiqrClient client = new SpecificBlockingRestKiqrClientImpl<>("localhost", 44321, "kv", String.class, Long.class, Serdes.String(), Serdes.Long()); 102 | 103 | //querying key "key1" from key-value store "kv" with String keys and Long values 104 | Optional result = client.getScalarKeyValue("key1"); 105 | 106 | //querying all keys from store "kv" with String keys and Long values 107 | Map result = client.getAllKeyValues(); 108 | 109 | //querying key range "key1" to "key3" from store "kv" with String keys and Long values 110 | Map result = client.getRangeKeyValues("key1", "key3"); 111 | 112 | //querying windows for "key1" from epoch time 1 to epoch time 1000 from store "window" with String keys and Long values 113 | Map result = client.getWindow("key1", 1L, 1000L); 114 | 115 | ``` 116 | 117 | ## Caveats and restrictions 118 | 119 | * not very well integrationally tested (yet? it is a hobby project) 120 | * not HA (when the streams app is rebalancing, there is not much you can do at this point) 121 | * No streaming of large results - if you query too much data, things will probably get weird. 122 | * highly unstable API and implementation, things will change 123 | * you are responsible to know the names of the state stores and types of your keys and values in Kafka. There is 124 | no way to infer them at runtime. 125 | * Java 8+ 126 | * Kafka Streams 0.10.2 127 | 128 | 129 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/SessionWindowQueryBlockingRestKiqrServiceImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.ConnectionException; 19 | import com.github.ftrossbach.kiqr.client.service.QueryExecutionException; 20 | import com.github.ftrossbach.kiqr.commons.config.Config; 21 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 22 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 23 | import io.vertx.core.Future; 24 | import io.vertx.core.http.HttpServerOptions; 25 | import io.vertx.ext.unit.Async; 26 | import io.vertx.ext.unit.TestContext; 27 | import io.vertx.ext.unit.junit.RunTestOnContext; 28 | import io.vertx.ext.unit.junit.VertxUnitRunner; 29 | import org.apache.kafka.common.serialization.Serdes; 30 | import org.junit.Before; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import java.util.*; 35 | 36 | /** 37 | * Created by ftr on 07/03/2017. 38 | */ 39 | @RunWith(VertxUnitRunner.class) 40 | public class SessionWindowQueryBlockingRestKiqrServiceImplTest { 41 | 42 | GenericBlockingRestKiqrClientImpl unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", 4567); 43 | 44 | @Rule 45 | public final RunTestOnContext rule = new RunTestOnContext(); 46 | 47 | String keyAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key1")); 48 | String keyAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key2")); 49 | 50 | 51 | String valueAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 42L)); 52 | String valueAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 4711L)); 53 | 54 | @Before 55 | public void setUp() { 56 | rule.vertx().eventBus().registerDefaultCodec(KeyBasedQuery.class, new KiqrCodec(KeyBasedQuery.class)); 57 | rule.vertx().eventBus().registerDefaultCodec(SessionQueryResponse.class, new KiqrCodec(SessionQueryResponse.class)); 58 | 59 | 60 | } 61 | 62 | 63 | @Test 64 | public void successfulQuery(TestContext context) { 65 | Async async = context.async(); 66 | 67 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 68 | 69 | rule.vertx().eventBus().consumer(Config.SESSION_QUERY_FACADE_ADDRESS, msg -> { 70 | 71 | 72 | 73 | 74 | 75 | List results = new ArrayList<>(); 76 | results.add(new Window(0,100, valueAsByteArray1)); 77 | results.add(new Window(101,200, valueAsByteArray2)); 78 | 79 | msg.reply(new SessionQueryResponse(results)); 80 | 81 | }); 82 | 83 | rule.vertx().executeBlocking((Future future) -> { 84 | 85 | Map result = unitUnderTest.getSession("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 86 | context.assertFalse(result.isEmpty()); 87 | context.assertEquals(2, result.size()); 88 | 89 | context.assertTrue(result.containsValue(4711L)); 90 | context.assertTrue(result.containsValue(42L)); 91 | async.complete(); 92 | 93 | future.complete(); 94 | }, context.asyncAssertSuccess()); 95 | 96 | })); 97 | 98 | } 99 | 100 | @Test 101 | public void notFound(TestContext context) { 102 | Async async = context.async(); 103 | 104 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 105 | 106 | rule.vertx().eventBus().consumer(Config.SESSION_QUERY_FACADE_ADDRESS, msg -> { 107 | 108 | msg.fail(404, "does not matter"); 109 | 110 | }); 111 | 112 | rule.vertx().executeBlocking(future -> { 113 | 114 | 115 | Map result = unitUnderTest.getSession("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 116 | context.assertTrue(result.isEmpty()); 117 | async.complete(); 118 | 119 | future.complete(); 120 | }, context.asyncAssertSuccess()); 121 | 122 | })); 123 | 124 | } 125 | 126 | @Test 127 | public void badRequest(TestContext context) { 128 | Async async = context.async(); 129 | 130 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 131 | 132 | rule.vertx().eventBus().consumer(Config.SESSION_QUERY_FACADE_ADDRESS, msg -> { 133 | 134 | msg.fail(400, "does not matter"); 135 | 136 | }); 137 | 138 | rule.vertx().executeBlocking(future -> { 139 | 140 | Map result = unitUnderTest.getSession("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 141 | context.fail(); 142 | 143 | 144 | }, context.asyncAssertFailure(ex -> { 145 | 146 | context.assertTrue(ex instanceof IllegalArgumentException); 147 | async.complete(); 148 | })); 149 | 150 | })); 151 | 152 | } 153 | 154 | @Test 155 | public void internalServerError(TestContext context) { 156 | Async async = context.async(); 157 | 158 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 159 | 160 | rule.vertx().eventBus().consumer(Config.SESSION_QUERY_FACADE_ADDRESS, msg -> { 161 | 162 | msg.fail(500, "does not matter"); 163 | 164 | }); 165 | 166 | rule.vertx().executeBlocking(future -> { 167 | 168 | Map result = unitUnderTest.getSession("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 169 | context.fail(); 170 | 171 | 172 | }, context.asyncAssertFailure(ex -> { 173 | context.assertTrue(ex instanceof QueryExecutionException); 174 | async.complete(); 175 | })); 176 | 177 | })); 178 | 179 | } 180 | 181 | 182 | 183 | } 184 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/RuntimeVerticleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core; 17 | 18 | import io.vertx.ext.unit.TestContext; 19 | import io.vertx.ext.unit.junit.RunTestOnContext; 20 | import io.vertx.ext.unit.junit.VertxUnitRunner; 21 | import org.apache.kafka.common.serialization.Serdes; 22 | import org.apache.kafka.streams.KafkaStreams; 23 | import org.apache.kafka.streams.StreamsConfig; 24 | import org.apache.kafka.streams.kstream.KStreamBuilder; 25 | import org.junit.Rule; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import java.util.Properties; 29 | import static org.junit.Assert.*; 30 | import static org.hamcrest.Matchers.*; 31 | 32 | 33 | import static org.mockito.Mockito.*; 34 | 35 | /** 36 | * Created by ftr on 06/03/2017. 37 | */ 38 | @RunWith(VertxUnitRunner.class) 39 | public class RuntimeVerticleTest { 40 | 41 | @Rule 42 | public RunTestOnContext rule = new RunTestOnContext(); 43 | 44 | @Test 45 | public void successfulStart(TestContext context){ 46 | 47 | KafkaStreams streamsMock = mock(KafkaStreams.class); 48 | KStreamBuilder builderMock = mock(KStreamBuilder.class); 49 | Properties props = new Properties(); 50 | RuntimeVerticle verticleSpy = spy(new RuntimeVerticle(builderMock, props, null)); 51 | 52 | doReturn(streamsMock).when(verticleSpy).createAndStartStream(); 53 | 54 | rule.vertx().deployVerticle(verticleSpy, context.asyncAssertSuccess(handler -> { 55 | 56 | 57 | context.assertTrue(rule.vertx().deploymentIDs().size() > 0); 58 | })); 59 | 60 | } 61 | 62 | @Test 63 | public void failingStart(TestContext context){ 64 | 65 | KafkaStreams streamsMock = mock(KafkaStreams.class); 66 | KStreamBuilder builderMock = mock(KStreamBuilder.class); 67 | Properties props = new Properties(); 68 | RuntimeVerticle verticleSpy = spy(new RuntimeVerticle(builderMock, props, null)); 69 | 70 | doThrow(RuntimeException.class).when(verticleSpy).createAndStartStream(); 71 | 72 | rule.vertx().deployVerticle(verticleSpy, context.asyncAssertFailure()); 73 | 74 | } 75 | 76 | 77 | @Test 78 | public void builderApplicationId(){ 79 | 80 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 81 | 82 | builder.withApplicationId("test"); 83 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 84 | 85 | assertThat(verticle.props, hasKey(StreamsConfig.APPLICATION_ID_CONFIG)); 86 | assertThat(verticle.props.get(StreamsConfig.APPLICATION_ID_CONFIG), is(equalTo("test"))); 87 | 88 | } 89 | 90 | @Test 91 | public void builderBootstrapServer(){ 92 | 93 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 94 | 95 | builder.withBootstrapServers("localhost:123"); 96 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 97 | 98 | assertThat(verticle.props, hasKey(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG)); 99 | assertThat(verticle.props.get(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG), is(equalTo("localhost:123"))); 100 | 101 | } 102 | 103 | @Test 104 | public void builderWithCaching(){ 105 | 106 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 107 | 108 | builder.withBuffering(42); 109 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 110 | 111 | assertThat(verticle.props, hasKey(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG)); 112 | assertThat(verticle.props.get(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG), is(equalTo(42))); 113 | 114 | } 115 | 116 | @Test 117 | public void builderWithoutCaching(){ 118 | 119 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 120 | 121 | builder.withoutBuffering(); 122 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 123 | 124 | assertThat(verticle.props, hasKey(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG)); 125 | assertThat(verticle.props.get(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG), is(equalTo(0))); 126 | 127 | } 128 | 129 | @Test 130 | public void builderWithKeySerdeString(){ 131 | 132 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 133 | 134 | builder.withKeySerde(Serdes.String()); 135 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 136 | 137 | assertThat(verticle.props, hasKey(StreamsConfig.KEY_SERDE_CLASS_CONFIG)); 138 | assertThat(verticle.props.get(StreamsConfig.KEY_SERDE_CLASS_CONFIG), is(equalTo(Serdes.String().getClass().getName()))); 139 | 140 | } 141 | 142 | @Test 143 | public void builderWithKeySerdeClass(){ 144 | 145 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 146 | 147 | builder.withKeySerde(Serdes.String().getClass()); 148 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 149 | 150 | assertThat(verticle.props, hasKey(StreamsConfig.KEY_SERDE_CLASS_CONFIG)); 151 | assertThat(verticle.props.get(StreamsConfig.KEY_SERDE_CLASS_CONFIG), is(equalTo(Serdes.String().getClass().getName()))); 152 | 153 | } 154 | 155 | @Test 156 | public void builderWithValueSerdeString(){ 157 | 158 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 159 | 160 | builder.withValueSerde(Serdes.String()); 161 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 162 | 163 | assertThat(verticle.props, hasKey(StreamsConfig.VALUE_SERDE_CLASS_CONFIG)); 164 | assertThat(verticle.props.get(StreamsConfig.VALUE_SERDE_CLASS_CONFIG), is(equalTo(Serdes.String().getClass().getName()))); 165 | 166 | } 167 | 168 | @Test 169 | public void builderWithValueSerdeClass(){ 170 | 171 | RuntimeVerticle.Builder builder = RuntimeVerticle.Builder.builder(new KStreamBuilder(), new Properties()); 172 | 173 | builder.withValueSerde(Serdes.String().getClass()); 174 | RuntimeVerticle verticle = (RuntimeVerticle) builder.build(); 175 | 176 | assertThat(verticle.props, hasKey(StreamsConfig.VALUE_SERDE_CLASS_CONFIG)); 177 | assertThat(verticle.props.get(StreamsConfig.VALUE_SERDE_CLASS_CONFIG), is(equalTo(Serdes.String().getClass().getName()))); 178 | 179 | } 180 | 181 | 182 | } 183 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/facade/WindowQueryFacadeVerticleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.facade; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.ShareableStreamsMetadataProvider; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.eventbus.ReplyException; 23 | import io.vertx.ext.unit.TestContext; 24 | import io.vertx.ext.unit.junit.RunTestOnContext; 25 | import io.vertx.ext.unit.junit.VertxUnitRunner; 26 | import org.apache.kafka.common.serialization.Serdes; 27 | import org.apache.kafka.common.serialization.Serializer; 28 | import org.apache.kafka.streams.state.HostInfo; 29 | import org.apache.kafka.streams.state.StreamsMetadata; 30 | import org.junit.Before; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import java.util.Collections; 35 | import java.util.Optional; 36 | import java.util.SortedMap; 37 | import java.util.TreeMap; 38 | 39 | 40 | import static org.mockito.Matchers.any; 41 | import static org.mockito.Matchers.anyString; 42 | import static org.mockito.Mockito.mock; 43 | import static org.mockito.Mockito.when; 44 | 45 | /** 46 | * Created by ftr on 06/03/2017. 47 | */ 48 | @RunWith(VertxUnitRunner.class) 49 | public class WindowQueryFacadeVerticleTest { 50 | 51 | @Rule 52 | public RunTestOnContext rule = new RunTestOnContext(); 53 | 54 | @Before 55 | public void setUp(){ 56 | rule.vertx().eventBus().registerDefaultCodec(WindowedQuery.class, new KiqrCodec(WindowedQuery.class)); 57 | rule.vertx().eventBus().registerDefaultCodec(WindowedQueryResponse.class, new KiqrCodec(WindowedQueryResponse.class)); 58 | 59 | } 60 | 61 | @Test 62 | public void success(TestContext context){ 63 | 64 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 65 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 66 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(host); 67 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 68 | 69 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_ADDRESS_PREFIX + "host", msg -> { 70 | SortedMap result = new TreeMap<>(); 71 | 72 | result.put(1L, "abc"); 73 | result.put(2L, "def"); 74 | msg.reply(new WindowedQueryResponse(result )); 75 | }); 76 | 77 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.WINDOWED_QUERY_FACADE_ADDRESS, Config.WINDOWED_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 78 | 79 | WindowedQuery query = new WindowedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(),Serdes.String().getClass().getName(), 1, 2); 80 | 81 | rule.vertx().eventBus().send(Config.WINDOWED_QUERY_FACADE_ADDRESS, query, context.asyncAssertSuccess(reply ->{ 82 | 83 | context.assertTrue(reply.body() instanceof WindowedQueryResponse); 84 | WindowedQueryResponse response = (WindowedQueryResponse) reply.body(); 85 | context.assertEquals(2, response.getValues().size()); 86 | context.assertTrue(response.getValues().containsKey(1L)); 87 | context.assertEquals("abc", response.getValues().get(1L)); 88 | context.assertTrue(response.getValues().containsKey(2L)); 89 | context.assertEquals("def", response.getValues().get(2L)); 90 | 91 | })); 92 | 93 | 94 | })); 95 | 96 | } 97 | 98 | @Test 99 | public void forwardingFailureDuringQuery(TestContext context){ 100 | 101 | 102 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 103 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 104 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(host); 105 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 106 | 107 | 108 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_ADDRESS_PREFIX + "host", msg -> { 109 | msg.fail(400, "msg"); 110 | }); 111 | 112 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.WINDOWED_QUERY_FACADE_ADDRESS, Config.WINDOWED_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 113 | 114 | WindowedQuery query = new WindowedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(),Serdes.String().getClass().getName(), 1, 2); 115 | 116 | rule.vertx().eventBus().send(Config.WINDOWED_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 117 | 118 | context.assertTrue(handler instanceof ReplyException); 119 | ReplyException ex = (ReplyException) handler; 120 | context.assertEquals(400, ex.failureCode()); 121 | context.assertEquals("msg", ex.getMessage()); 122 | 123 | })); 124 | })); 125 | 126 | } 127 | 128 | @Test 129 | public void forwardingFailureDuringInstanceLookup(TestContext context){ 130 | 131 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 132 | StreamsMetadata host = new StreamsMetadata(new HostInfo("host", 1), Collections.emptySet(), Collections.emptySet()); 133 | when(mock.metadataForKey(anyString(), any(), any(Serializer.class))).thenReturn(null); 134 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 135 | 136 | 137 | rule.vertx().deployVerticle(new KeyBasedQueryFacadeVerticle(Config.WINDOWED_QUERY_FACADE_ADDRESS, Config.WINDOWED_QUERY_ADDRESS_PREFIX), context.asyncAssertSuccess(deployment->{ 138 | 139 | WindowedQuery query = new WindowedQuery("store", Serdes.String().getClass().getName(), "key".getBytes(),Serdes.String().getClass().getName(), 1, 2); 140 | 141 | rule.vertx().eventBus().send(Config.WINDOWED_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 142 | 143 | context.assertTrue(handler instanceof ReplyException); 144 | ReplyException ex = (ReplyException) handler; 145 | context.assertEquals(404, ex.failureCode()); 146 | 147 | })); 148 | })); 149 | 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/WindowedQueryHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.RuntimeVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.http.HttpServerOptions; 23 | import io.vertx.ext.unit.Async; 24 | import io.vertx.ext.unit.TestContext; 25 | import io.vertx.ext.unit.junit.RunTestOnContext; 26 | import io.vertx.ext.unit.junit.VertxUnitRunner; 27 | import org.apache.kafka.common.serialization.Serdes; 28 | import org.junit.Before; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | import org.junit.runner.RunWith; 32 | import java.util.*; 33 | 34 | 35 | import static org.mockito.Mockito.mock; 36 | 37 | /** 38 | * Created by ftr on 06/03/2017. 39 | */ 40 | @RunWith(VertxUnitRunner.class) 41 | public class WindowedQueryHttpServerTest { 42 | 43 | @Rule 44 | public final RunTestOnContext rule = new RunTestOnContext(); 45 | 46 | 47 | 48 | @Before 49 | public void setUp(TestContext context) throws Exception{ 50 | rule.vertx().eventBus().registerDefaultCodec(WindowedQuery.class, new KiqrCodec(WindowedQuery.class)); 51 | rule.vertx().eventBus().registerDefaultCodec(WindowedQueryResponse.class, new KiqrCodec<>(WindowedQueryResponse.class)); 52 | 53 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummySuccessfulVerticle()), context.asyncAssertSuccess()); 54 | } 55 | 56 | 57 | @Test 58 | public void readWindowValue(TestContext context) throws Exception { 59 | 60 | Async async = context.async(); 61 | 62 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 63 | 64 | 65 | 66 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 67 | context.assertTrue(msg.body() instanceof WindowedQuery); 68 | WindowedQuery query = (WindowedQuery) msg.body(); 69 | 70 | context.assertEquals("store", query.getStoreName()); 71 | context.assertEquals(Serdes.String().getClass().getName(), query.getKeySerde()); 72 | context.assertEquals(Serdes.Long().getClass().getName(), query.getValueSerde()); 73 | context.assertTrue(query.getKey().length > 0); 74 | context.assertEquals(1L, query.getFrom()); 75 | context.assertEquals(2L, query.getTo()); 76 | 77 | SortedMap results = new TreeMap<>(); 78 | results.put(1L, "value1"); 79 | results.put(2L, "value2"); 80 | 81 | msg.reply(new WindowedQueryResponse(results)); 82 | 83 | }); 84 | 85 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&valueSerde=%s&from=1&to=2", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 86 | 87 | context.assertEquals(200, res.statusCode()); 88 | async.complete(); 89 | }).end(); 90 | } 91 | 92 | @Test 93 | public void valuesEmpty(TestContext context) { 94 | 95 | Async async = context.async(); 96 | 97 | 98 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 99 | 100 | msg.reply(new WindowedQueryResponse(Collections.emptySortedMap())); 101 | 102 | }); 103 | 104 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&valueSerde=%s&from=1&to=2", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 105 | 106 | context.assertEquals(200, res.statusCode()); 107 | async.complete(); 108 | }).end(); 109 | } 110 | 111 | 112 | 113 | 114 | 115 | @Test 116 | public void noKeySerde(TestContext context) { 117 | 118 | Async async = context.async(); 119 | 120 | 121 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?valueSerde=%s&from=1&to=2", Serdes.Long().getClass().getName()), res ->{ 122 | 123 | context.assertEquals(400, res.statusCode()); 124 | async.complete(); 125 | }).end(); 126 | } 127 | 128 | @Test 129 | public void noValueSerde(TestContext context) { 130 | 131 | Async async = context.async(); 132 | 133 | 134 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&from=1&to=2", Serdes.Long().getClass().getName()), res ->{ 135 | 136 | context.assertEquals(400, res.statusCode()); 137 | async.complete(); 138 | }).end(); 139 | } 140 | 141 | @Test 142 | public void internalServerError(TestContext context){ 143 | Async async = context.async(); 144 | 145 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 146 | 147 | 148 | 149 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 150 | msg.fail(500, "msg"); 151 | 152 | }); 153 | 154 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&valueSerde=%s&from=1&to=2", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 155 | 156 | context.assertEquals(500, res.statusCode()); 157 | async.complete(); 158 | }).end(); 159 | } 160 | 161 | @Test 162 | public void noFrom(TestContext context){ 163 | Async async = context.async(); 164 | 165 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 166 | 167 | 168 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&valueSerde=%s&to=2", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 169 | 170 | context.assertEquals(400, res.statusCode()); 171 | async.complete(); 172 | }).end(); 173 | } 174 | 175 | @Test 176 | public void noTo(TestContext context){ 177 | Async async = context.async(); 178 | 179 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 180 | 181 | 182 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/window/store/key?keySerde=%s&valueSerde=%s&from=1", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 183 | 184 | context.assertEquals(400, res.statusCode()); 185 | async.complete(); 186 | }).end(); 187 | } 188 | 189 | 190 | 191 | 192 | } 193 | -------------------------------------------------------------------------------- /rest-server/src/test/java/com/github/ftrossbach/kiqr/rest/server/MultiValuedQueryHttpServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.rest.server; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.RuntimeVerticle; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.http.HttpServerOptions; 23 | import io.vertx.ext.unit.Async; 24 | import io.vertx.ext.unit.TestContext; 25 | import io.vertx.ext.unit.junit.RunTestOnContext; 26 | import io.vertx.ext.unit.junit.VertxUnitRunner; 27 | import org.apache.kafka.common.serialization.Serdes; 28 | import org.junit.Before; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | import org.junit.runner.RunWith; 32 | 33 | 34 | import java.util.Collections; 35 | import java.util.HashMap; 36 | import java.util.Map; 37 | 38 | 39 | import static org.mockito.Mockito.mock; 40 | 41 | /** 42 | * Created by ftr on 06/03/2017. 43 | */ 44 | @RunWith(VertxUnitRunner.class) 45 | public class MultiValuedQueryHttpServerTest { 46 | 47 | @Rule 48 | public final RunTestOnContext rule = new RunTestOnContext(); 49 | 50 | 51 | 52 | @Before 53 | public void setUp(TestContext context) throws Exception{ 54 | rule.vertx().eventBus().registerDefaultCodec(StoreWideQuery.class, new KiqrCodec(StoreWideQuery.class)); 55 | rule.vertx().eventBus().registerDefaultCodec(RangeKeyValueQuery.class, new KiqrCodec<>(RangeKeyValueQuery.class)); 56 | rule.vertx().eventBus().registerDefaultCodec(MultiValuedKeyValueQueryResponse.class, new KiqrCodec(MultiValuedKeyValueQueryResponse.class)); 57 | 58 | 59 | rule.vertx().deployVerticle(new RestKiqrServerVerticle(new HttpServerOptions().setPort(5762), new DummySuccessfulVerticle()), context.asyncAssertSuccess()); 60 | } 61 | 62 | 63 | @Test 64 | public void readAllValue(TestContext context) throws Exception { 65 | 66 | Async async = context.async(); 67 | 68 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 69 | 70 | 71 | 72 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 73 | context.assertTrue(msg.body() instanceof StoreWideQuery); 74 | StoreWideQuery query = (StoreWideQuery) msg.body(); 75 | 76 | context.assertEquals("store", query.getStoreName()); 77 | context.assertEquals(Serdes.String().getClass().getName(), query.getKeySerde()); 78 | context.assertEquals(Serdes.Long().getClass().getName(), query.getValueSerde()); 79 | 80 | Map results = new HashMap<>(); 81 | results.put("key1", "value1"); 82 | results.put("key2", "value2"); 83 | 84 | msg.reply(new MultiValuedKeyValueQueryResponse(results)); 85 | 86 | }); 87 | 88 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 89 | 90 | context.assertEquals(200, res.statusCode()); 91 | async.complete(); 92 | }).end(); 93 | } 94 | 95 | @Test 96 | public void allKValuesEmpty(TestContext context) { 97 | 98 | Async async = context.async(); 99 | 100 | 101 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 102 | 103 | msg.reply(new MultiValuedKeyValueQueryResponse(Collections.emptyMap())); 104 | 105 | }); 106 | 107 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 108 | 109 | context.assertEquals(200, res.statusCode()); 110 | async.complete(); 111 | }).end(); 112 | } 113 | 114 | 115 | 116 | @Test 117 | public void readRangeValue(TestContext context) throws Exception { 118 | 119 | Async async = context.async(); 120 | 121 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 122 | 123 | 124 | 125 | rule.vertx().eventBus().consumer(Config.RANGE_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 126 | context.assertTrue(msg.body() instanceof RangeKeyValueQuery); 127 | RangeKeyValueQuery query = (RangeKeyValueQuery) msg.body(); 128 | 129 | context.assertEquals("store", query.getStoreName()); 130 | context.assertEquals(Serdes.String().getClass().getName(), query.getKeySerde()); 131 | context.assertEquals(Serdes.Long().getClass().getName(), query.getValueSerde()); 132 | context.assertTrue(query.getFrom().length > 0); 133 | context.assertTrue(query.getTo().length > 0); 134 | 135 | Map results = new HashMap<>(); 136 | results.put("key1", "value1"); 137 | results.put("key2", "value2"); 138 | 139 | msg.reply(new MultiValuedKeyValueQueryResponse(results)); 140 | 141 | }); 142 | 143 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s&valueSerde=%s&from=from&to=to", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 144 | 145 | context.assertEquals(200, res.statusCode()); 146 | async.complete(); 147 | }).end(); 148 | } 149 | 150 | @Test 151 | public void rangeEmpty(TestContext context) { 152 | 153 | Async async = context.async(); 154 | 155 | 156 | rule.vertx().eventBus().consumer(Config.RANGE_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 157 | 158 | msg.reply(new MultiValuedKeyValueQueryResponse(Collections.emptyMap())); 159 | 160 | }); 161 | 162 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s&valueSerde=%s&from=from&to=to", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 163 | 164 | context.assertEquals(200, res.statusCode()); 165 | async.complete(); 166 | }).end(); 167 | } 168 | 169 | 170 | @Test 171 | public void noKeySerde(TestContext context) { 172 | 173 | Async async = context.async(); 174 | 175 | 176 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?valueSerde=%s", Serdes.Long().getClass().getName()), res ->{ 177 | 178 | context.assertEquals(400, res.statusCode()); 179 | async.complete(); 180 | }).end(); 181 | } 182 | 183 | @Test 184 | public void noValueSerde(TestContext context) { 185 | 186 | Async async = context.async(); 187 | 188 | 189 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s", Serdes.Long().getClass().getName()), res ->{ 190 | 191 | context.assertEquals(400, res.statusCode()); 192 | async.complete(); 193 | }).end(); 194 | } 195 | 196 | @Test 197 | public void internalServerError(TestContext context){ 198 | Async async = context.async(); 199 | 200 | RuntimeVerticle mock = mock(RuntimeVerticle.class); 201 | 202 | 203 | 204 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 205 | msg.fail(500, "msg"); 206 | 207 | }); 208 | 209 | rule.vertx().createHttpClient().get(5762, "localhost", String.format("/api/v1/kv/store?keySerde=%s&valueSerde=%s", Serdes.String().getClass().getName(), Serdes.Long().getClass().getName()), res ->{ 210 | 211 | context.assertEquals(500, res.statusCode()); 212 | async.complete(); 213 | }).end(); 214 | } 215 | 216 | 217 | 218 | 219 | } 220 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/ScalarKVQueryBlockingRestKiqrServiceImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.ConnectionException; 19 | import com.github.ftrossbach.kiqr.client.service.QueryExecutionException; 20 | import com.github.ftrossbach.kiqr.commons.config.Config; 21 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.KeyBasedQuery; 22 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.ScalarKeyValueQueryResponse; 23 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 24 | import io.vertx.core.http.HttpServerOptions; 25 | import io.vertx.ext.unit.Async; 26 | import io.vertx.ext.unit.TestContext; 27 | import io.vertx.ext.unit.junit.RunTestOnContext; 28 | import io.vertx.ext.unit.junit.VertxUnitRunner; 29 | import org.apache.kafka.common.serialization.Serdes; 30 | import org.junit.Before; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import java.util.Base64; 35 | import java.util.Optional; 36 | 37 | /** 38 | * Created by ftr on 07/03/2017. 39 | */ 40 | @RunWith(VertxUnitRunner.class) 41 | public class ScalarKVQueryBlockingRestKiqrServiceImplTest { 42 | 43 | GenericBlockingRestKiqrClientImpl unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", 4567); 44 | 45 | @Rule 46 | public final RunTestOnContext rule = new RunTestOnContext(); 47 | 48 | String valueAsByteArray = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 42L)); 49 | 50 | @Before 51 | public void setUp(){ 52 | rule.vertx().eventBus().registerDefaultCodec(KeyBasedQuery.class, new KiqrCodec(KeyBasedQuery.class)); 53 | rule.vertx().eventBus().registerDefaultCodec(ScalarKeyValueQueryResponse.class, new KiqrCodec(ScalarKeyValueQueryResponse.class)); 54 | 55 | 56 | } 57 | 58 | 59 | @Test 60 | public void successfulQuery(TestContext context){ 61 | Async async = context.async(); 62 | 63 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 64 | 65 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 66 | 67 | msg.reply(new ScalarKeyValueQueryResponse(valueAsByteArray)); 68 | 69 | }); 70 | 71 | rule.vertx().executeBlocking(future -> { 72 | 73 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 74 | context.assertTrue(scalarKeyValue.isPresent()); 75 | context.assertEquals(42L, scalarKeyValue.get()); 76 | async.complete(); 77 | 78 | future.complete(); 79 | }, context.asyncAssertSuccess()); 80 | 81 | })); 82 | 83 | } 84 | 85 | @Test 86 | public void notFound(TestContext context){ 87 | Async async = context.async(); 88 | 89 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 90 | 91 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 92 | 93 | msg.fail(404, "does not matter"); 94 | 95 | }); 96 | 97 | rule.vertx().executeBlocking(future -> { 98 | 99 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 100 | context.assertFalse(scalarKeyValue.isPresent()); 101 | async.complete(); 102 | 103 | future.complete(); 104 | }, context.asyncAssertSuccess()); 105 | 106 | })); 107 | 108 | } 109 | 110 | @Test 111 | public void badRequest(TestContext context){ 112 | Async async = context.async(); 113 | 114 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 115 | 116 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 117 | 118 | msg.fail(400, "does not matter"); 119 | 120 | }); 121 | 122 | rule.vertx().executeBlocking(future -> { 123 | 124 | 125 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 126 | context.fail(); 127 | 128 | 129 | 130 | 131 | }, context.asyncAssertFailure(ex -> { 132 | 133 | context.assertTrue(ex instanceof IllegalArgumentException); 134 | async.complete(); 135 | })); 136 | 137 | })); 138 | 139 | } 140 | 141 | @Test 142 | public void internalServerError(TestContext context){ 143 | Async async = context.async(); 144 | 145 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 146 | 147 | rule.vertx().eventBus().consumer(Config.KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 148 | 149 | msg.fail(500, "does not matter"); 150 | 151 | }); 152 | 153 | rule.vertx().executeBlocking(future -> { 154 | 155 | 156 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 157 | context.fail(); 158 | 159 | 160 | 161 | }, context.asyncAssertFailure(ex -> { 162 | context.assertTrue(ex instanceof QueryExecutionException); 163 | async.complete(); 164 | })); 165 | 166 | })); 167 | 168 | } 169 | 170 | @Test 171 | public void connectionExceptionInvalidPort(TestContext context){ 172 | 173 | 174 | Async async = context.async(); 175 | 176 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 177 | 178 | rule.vertx().executeBlocking(future -> { 179 | 180 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", -1); 181 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 182 | context.fail(); 183 | 184 | 185 | 186 | }, context.asyncAssertFailure(ex -> { 187 | context.assertTrue(ex instanceof ConnectionException); 188 | async.complete(); 189 | })); 190 | 191 | })); 192 | 193 | } 194 | 195 | @Test 196 | public void connectionExceptionInvalidHost(TestContext context){ 197 | Async async = context.async(); 198 | 199 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 200 | 201 | rule.vertx().executeBlocking(future -> { 202 | 203 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("host with spaces", 4567); 204 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 205 | context.fail(); 206 | 207 | 208 | 209 | }, context.asyncAssertFailure(ex -> { 210 | context.assertTrue(ex instanceof ConnectionException); 211 | async.complete(); 212 | })); 213 | 214 | })); 215 | 216 | } 217 | 218 | @Test 219 | public void validButUnReachableHost(TestContext context){ 220 | Async async = context.async(); 221 | 222 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 223 | 224 | rule.vertx().executeBlocking(future -> { 225 | 226 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("ihopethisdoesntexist", 4567); 227 | Optional scalarKeyValue = unitUnderTest.getScalarKeyValue("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long()); 228 | context.fail(); 229 | 230 | 231 | 232 | }, context.asyncAssertFailure(ex -> { 233 | context.assertTrue(ex instanceof ConnectionException); 234 | async.complete(); 235 | })); 236 | 237 | })); 238 | 239 | } 240 | 241 | } 242 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/WindowQueryBlockingRestKiqrServiceImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.ConnectionException; 19 | import com.github.ftrossbach.kiqr.client.service.QueryExecutionException; 20 | import com.github.ftrossbach.kiqr.commons.config.Config; 21 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.WindowedQuery; 22 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.WindowedQueryResponse; 23 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 24 | import io.vertx.core.Future; 25 | import io.vertx.core.http.HttpServerOptions; 26 | import io.vertx.ext.unit.Async; 27 | import io.vertx.ext.unit.TestContext; 28 | import io.vertx.ext.unit.junit.RunTestOnContext; 29 | import io.vertx.ext.unit.junit.VertxUnitRunner; 30 | import org.apache.kafka.common.serialization.Serdes; 31 | import org.junit.Before; 32 | import org.junit.Rule; 33 | import org.junit.Test; 34 | import org.junit.runner.RunWith; 35 | import java.util.*; 36 | 37 | /** 38 | * Created by ftr on 07/03/2017. 39 | */ 40 | @RunWith(VertxUnitRunner.class) 41 | public class WindowQueryBlockingRestKiqrServiceImplTest { 42 | 43 | GenericBlockingRestKiqrClientImpl unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", 4567); 44 | 45 | @Rule 46 | public final RunTestOnContext rule = new RunTestOnContext(); 47 | 48 | String keyAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key1")); 49 | String keyAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key2")); 50 | 51 | 52 | String valueAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 42L)); 53 | String valueAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 4711L)); 54 | 55 | @Before 56 | public void setUp() { 57 | rule.vertx().eventBus().registerDefaultCodec(WindowedQuery.class, new KiqrCodec(WindowedQuery.class)); 58 | rule.vertx().eventBus().registerDefaultCodec(WindowedQueryResponse.class, new KiqrCodec(WindowedQueryResponse.class)); 59 | 60 | 61 | } 62 | 63 | 64 | @Test 65 | public void successfulQuery(TestContext context) { 66 | Async async = context.async(); 67 | 68 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 69 | 70 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 71 | 72 | 73 | SortedMap result = new TreeMap<>(); 74 | result.put(1L, valueAsByteArray1); 75 | result.put(2L, valueAsByteArray2); 76 | 77 | 78 | msg.reply(new WindowedQueryResponse(result)); 79 | 80 | }); 81 | 82 | rule.vertx().executeBlocking((Future future) -> { 83 | 84 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 85 | context.assertFalse(result.isEmpty()); 86 | context.assertEquals(2, result.size()); 87 | context.assertNotNull(result.get(1L)); 88 | context.assertEquals(42L, result.get(1L)); 89 | context.assertNotNull(result.get(2L)); 90 | context.assertEquals(4711L, result.get(2L)); 91 | async.complete(); 92 | 93 | future.complete(); 94 | }, context.asyncAssertSuccess()); 95 | 96 | })); 97 | 98 | } 99 | 100 | @Test 101 | public void notFound(TestContext context) { 102 | Async async = context.async(); 103 | 104 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 105 | 106 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 107 | 108 | msg.fail(404, "does not matter"); 109 | 110 | }); 111 | 112 | rule.vertx().executeBlocking(future -> { 113 | 114 | 115 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 116 | context.assertTrue(result.isEmpty()); 117 | async.complete(); 118 | 119 | future.complete(); 120 | }, context.asyncAssertSuccess()); 121 | 122 | })); 123 | 124 | } 125 | 126 | @Test 127 | public void badRequest(TestContext context) { 128 | Async async = context.async(); 129 | 130 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 131 | 132 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 133 | 134 | msg.fail(400, "does not matter"); 135 | 136 | }); 137 | 138 | rule.vertx().executeBlocking(future -> { 139 | 140 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 141 | context.fail(); 142 | 143 | 144 | }, context.asyncAssertFailure(ex -> { 145 | 146 | context.assertTrue(ex instanceof IllegalArgumentException); 147 | async.complete(); 148 | })); 149 | 150 | })); 151 | 152 | } 153 | 154 | @Test 155 | public void internalServerError(TestContext context) { 156 | Async async = context.async(); 157 | 158 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 159 | 160 | rule.vertx().eventBus().consumer(Config.WINDOWED_QUERY_FACADE_ADDRESS, msg -> { 161 | 162 | msg.fail(500, "does not matter"); 163 | 164 | }); 165 | 166 | rule.vertx().executeBlocking(future -> { 167 | 168 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 169 | context.fail(); 170 | 171 | 172 | }, context.asyncAssertFailure(ex -> { 173 | context.assertTrue(ex instanceof QueryExecutionException); 174 | async.complete(); 175 | })); 176 | 177 | })); 178 | 179 | } 180 | 181 | @Test 182 | public void connectionExceptionInvalidPort(TestContext context) { 183 | 184 | 185 | Async async = context.async(); 186 | 187 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 188 | 189 | rule.vertx().executeBlocking(future -> { 190 | 191 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", -1); 192 | 193 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 194 | context.fail(); 195 | 196 | 197 | }, context.asyncAssertFailure(ex -> { 198 | context.assertTrue(ex instanceof ConnectionException); 199 | async.complete(); 200 | })); 201 | 202 | })); 203 | 204 | } 205 | 206 | @Test 207 | public void connectionExceptionInvalidHost(TestContext context) { 208 | Async async = context.async(); 209 | 210 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 211 | 212 | rule.vertx().executeBlocking(future -> { 213 | 214 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("host with spaces", 4567); 215 | 216 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 217 | context.fail(); 218 | 219 | 220 | }, context.asyncAssertFailure(ex -> { 221 | context.assertTrue(ex instanceof ConnectionException); 222 | async.complete(); 223 | })); 224 | 225 | })); 226 | 227 | } 228 | 229 | @Test 230 | public void validButUnReachableHost(TestContext context) { 231 | Async async = context.async(); 232 | 233 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 234 | 235 | rule.vertx().executeBlocking(future -> { 236 | 237 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("ihopethisdoesntexist", 4567); 238 | Map result = unitUnderTest.getWindow("store", String.class, "key", Long.class, Serdes.String(), Serdes.Long(), 1L, 2L); 239 | context.fail(); 240 | 241 | 242 | }, context.asyncAssertFailure(ex -> { 243 | context.assertTrue(ex instanceof ConnectionException); 244 | async.complete(); 245 | })); 246 | 247 | })); 248 | 249 | } 250 | 251 | } 252 | -------------------------------------------------------------------------------- /core/src/test/java/com/github/ftrossbach/kiqr/core/query/facade/AllKeyValueQueryFacadeVerticleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.core.query.facade; 17 | 18 | import com.github.ftrossbach.kiqr.commons.config.Config; 19 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.*; 20 | import com.github.ftrossbach.kiqr.core.ShareableStreamsMetadataProvider; 21 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 22 | import io.vertx.core.eventbus.ReplyException; 23 | import io.vertx.ext.unit.TestContext; 24 | import io.vertx.ext.unit.junit.RunTestOnContext; 25 | import io.vertx.ext.unit.junit.VertxUnitRunner; 26 | import org.apache.kafka.common.serialization.Serdes; 27 | import org.apache.kafka.streams.state.HostInfo; 28 | import org.apache.kafka.streams.state.StreamsMetadata; 29 | import org.junit.Before; 30 | import org.junit.Rule; 31 | import org.junit.Test; 32 | import org.junit.runner.RunWith; 33 | import static org.mockito.Mockito.*; 34 | import java.util.*; 35 | 36 | /** 37 | * Created by ftr on 05/03/2017. 38 | */ 39 | @RunWith(VertxUnitRunner.class) 40 | public class AllKeyValueQueryFacadeVerticleTest { 41 | 42 | @Rule 43 | public RunTestOnContext rule = new RunTestOnContext(); 44 | 45 | @Before 46 | public void setUp(){ 47 | rule.vertx().eventBus().registerDefaultCodec(AllInstancesResponse.class, new KiqrCodec(AllInstancesResponse.class)); 48 | rule.vertx().eventBus().registerDefaultCodec(StoreWideQuery.class, new KiqrCodec(StoreWideQuery.class)); 49 | rule.vertx().eventBus().registerDefaultCodec(MultiValuedKeyValueQueryResponse.class, new KiqrCodec(MultiValuedKeyValueQueryResponse.class)); 50 | 51 | } 52 | 53 | @Test 54 | public void successOneInstances(TestContext context){ 55 | 56 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 57 | 58 | StreamsMetadata host1 = new StreamsMetadata(new HostInfo("host1", 1), Collections.emptySet(), Collections.emptySet()); 59 | 60 | 61 | when(mock.allMetadataForStore(anyString())).thenReturn( Arrays.asList(new StreamsMetadata[]{host1})); 62 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 63 | 64 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host1", msg -> { 65 | Map result = new HashMap<>(); 66 | result.put("key1", "value1"); 67 | msg.reply(new MultiValuedKeyValueQueryResponse(result)); 68 | }); 69 | 70 | 71 | rule.vertx().deployVerticle(new ScatterGatherQueryFacadeVerticle(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, () -> new MultiValuedKeyValueQueryResponse(), (a, b) -> a.merge(b)), context.asyncAssertSuccess(deployment->{ 72 | 73 | StoreWideQuery query = new StoreWideQuery("store", Serdes.String().getClass().getName(), Serdes.String().getClass().getName()); 74 | 75 | rule.vertx().eventBus().send(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertSuccess(reply ->{ 76 | 77 | context.assertTrue(reply.body() instanceof MultiValuedKeyValueQueryResponse); 78 | MultiValuedKeyValueQueryResponse response = (MultiValuedKeyValueQueryResponse) reply.body(); 79 | context.assertEquals(1, response.getResults().size()); 80 | context.assertTrue(response.getResults().containsKey("key1")); 81 | context.assertEquals("value1", response.getResults().get("key1")); 82 | 83 | })); 84 | 85 | 86 | })); 87 | 88 | } 89 | 90 | @Test 91 | public void successTwoInstances(TestContext context){ 92 | 93 | 94 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 95 | 96 | StreamsMetadata host1 = new StreamsMetadata(new HostInfo("host1", 1), Collections.emptySet(), Collections.emptySet()); 97 | StreamsMetadata host2 = new StreamsMetadata(new HostInfo("host2", 1), Collections.emptySet(), Collections.emptySet()); 98 | 99 | when(mock.allMetadataForStore(anyString())).thenReturn( Arrays.asList(new StreamsMetadata[]{host1, host2})); 100 | 101 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 102 | 103 | 104 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host1", msg -> { 105 | Map result = new HashMap<>(); 106 | result.put("key1", "value1"); 107 | msg.reply(new MultiValuedKeyValueQueryResponse(result)); 108 | }); 109 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host2", msg -> { 110 | Map result = new HashMap<>(); 111 | result.put("key2", "value2"); 112 | msg.reply(new MultiValuedKeyValueQueryResponse(result)); 113 | }); 114 | 115 | rule.vertx().deployVerticle(new ScatterGatherQueryFacadeVerticle(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, () -> new MultiValuedKeyValueQueryResponse(), (a, b) -> a.merge(b)), context.asyncAssertSuccess(deployment->{ 116 | 117 | StoreWideQuery query = new StoreWideQuery("store", Serdes.String().getClass().getName(), Serdes.String().getClass().getName()); 118 | 119 | rule.vertx().eventBus().send(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertSuccess(reply ->{ 120 | 121 | context.assertTrue(reply.body() instanceof MultiValuedKeyValueQueryResponse); 122 | MultiValuedKeyValueQueryResponse response = (MultiValuedKeyValueQueryResponse) reply.body(); 123 | context.assertEquals(2, response.getResults().size()); 124 | context.assertTrue(response.getResults().containsKey("key1")); 125 | context.assertEquals("value1", response.getResults().get("key1")); 126 | context.assertTrue(response.getResults().containsKey("key2")); 127 | context.assertEquals("value2", response.getResults().get("key2")); 128 | 129 | })); 130 | 131 | 132 | })); 133 | 134 | } 135 | 136 | @Test 137 | public void failureOneSourceFails(TestContext context){ 138 | 139 | 140 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 141 | 142 | StreamsMetadata host1 = new StreamsMetadata(new HostInfo("host1", 1), Collections.emptySet(), Collections.emptySet()); 143 | StreamsMetadata host2 = new StreamsMetadata(new HostInfo("host2", 1), Collections.emptySet(), Collections.emptySet()); 144 | 145 | when(mock.allMetadataForStore(anyString())).thenReturn( Arrays.asList(new StreamsMetadata[]{host1, host2})); 146 | 147 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 148 | 149 | 150 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host1", msg -> { 151 | Map result = new HashMap<>(); 152 | result.put("key1", "value1"); 153 | msg.reply(new MultiValuedKeyValueQueryResponse(result)); 154 | }); 155 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX + "host2", msg -> { 156 | msg.fail(400, "msg"); 157 | }); 158 | 159 | rule.vertx().deployVerticle(new ScatterGatherQueryFacadeVerticle(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, () -> new MultiValuedKeyValueQueryResponse(), (a, b) -> a.merge(b)), context.asyncAssertSuccess(deployment->{ 160 | 161 | StoreWideQuery query = new StoreWideQuery("store", Serdes.String().getClass().getName(), Serdes.String().getClass().getName()); 162 | 163 | rule.vertx().eventBus().send(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 164 | 165 | context.assertTrue(handler instanceof ReplyException); 166 | ReplyException ex = (ReplyException) handler; 167 | context.assertEquals(400, ex.failureCode()); 168 | 169 | })); 170 | 171 | 172 | })); 173 | 174 | } 175 | 176 | @Test 177 | public void noInstanceFound(TestContext context){ 178 | 179 | 180 | ShareableStreamsMetadataProvider mock = mock(ShareableStreamsMetadataProvider.class); 181 | when(mock.allMetadataForStore(anyString())).thenReturn( Collections.emptyList()); 182 | rule.vertx().sharedData().getLocalMap("metadata").put("metadata", mock); 183 | 184 | 185 | 186 | rule.vertx().deployVerticle(new ScatterGatherQueryFacadeVerticle(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, () -> new MultiValuedKeyValueQueryResponse(), (a, b) -> a.merge(b)), context.asyncAssertSuccess(deployment->{ 187 | 188 | StoreWideQuery query = new StoreWideQuery("store", Serdes.String().getClass().getName(), Serdes.String().getClass().getName()); 189 | 190 | rule.vertx().eventBus().send(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, query, context.asyncAssertFailure(handler ->{ 191 | 192 | context.assertTrue(handler instanceof ReplyException); 193 | ReplyException ex = (ReplyException) handler; 194 | context.assertEquals(404, ex.failureCode()); 195 | 196 | })); 197 | 198 | 199 | })); 200 | 201 | } 202 | 203 | 204 | 205 | } 206 | -------------------------------------------------------------------------------- /rest-client/src/test/java/com/github/ftrossbach/kiqr/client/service/rest/AllKVQueryBlockingRestKiqrServiceImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2017 Florian Troßbach (trossbach@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.ftrossbach.kiqr.client.service.rest; 17 | 18 | import com.github.ftrossbach.kiqr.client.service.ConnectionException; 19 | import com.github.ftrossbach.kiqr.client.service.QueryExecutionException; 20 | import com.github.ftrossbach.kiqr.commons.config.Config; 21 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.StoreWideQuery; 22 | import com.github.ftrossbach.kiqr.commons.config.querymodel.requests.MultiValuedKeyValueQueryResponse; 23 | import com.github.ftrossbach.kiqr.core.query.KiqrCodec; 24 | import io.vertx.core.http.HttpServerOptions; 25 | import io.vertx.ext.unit.Async; 26 | import io.vertx.ext.unit.TestContext; 27 | import io.vertx.ext.unit.junit.RunTestOnContext; 28 | import io.vertx.ext.unit.junit.VertxUnitRunner; 29 | import org.apache.kafka.common.serialization.Serdes; 30 | import org.junit.Before; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import java.util.Base64; 35 | import java.util.HashMap; 36 | import java.util.Map; 37 | 38 | /** 39 | * Created by ftr on 07/03/2017. 40 | */ 41 | @RunWith(VertxUnitRunner.class) 42 | public class AllKVQueryBlockingRestKiqrServiceImplTest { 43 | 44 | GenericBlockingRestKiqrClientImpl unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", 4567); 45 | 46 | @Rule 47 | public final RunTestOnContext rule = new RunTestOnContext(); 48 | 49 | String keyAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key1")); 50 | String keyAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.String().serializer().serialize("", "key2")); 51 | 52 | 53 | String valueAsByteArray1 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 42L)); 54 | String valueAsByteArray2 = Base64.getEncoder().encodeToString(Serdes.Long().serializer().serialize("", 4711L)); 55 | 56 | @Before 57 | public void setUp() { 58 | rule.vertx().eventBus().registerDefaultCodec(StoreWideQuery.class, new KiqrCodec(StoreWideQuery.class)); 59 | rule.vertx().eventBus().registerDefaultCodec(MultiValuedKeyValueQueryResponse.class, new KiqrCodec(MultiValuedKeyValueQueryResponse.class)); 60 | 61 | 62 | } 63 | 64 | 65 | @Test 66 | public void successfulQuery(TestContext context) { 67 | Async async = context.async(); 68 | 69 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 70 | 71 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 72 | 73 | 74 | Map result = new HashMap<>(); 75 | result.put(keyAsByteArray1, valueAsByteArray1); 76 | result.put(keyAsByteArray2, valueAsByteArray2); 77 | 78 | 79 | msg.reply(new MultiValuedKeyValueQueryResponse(result)); 80 | 81 | }); 82 | 83 | rule.vertx().executeBlocking(future -> { 84 | 85 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 86 | context.assertFalse(result.isEmpty()); 87 | context.assertEquals(2, result.size()); 88 | context.assertNotNull(result.get("key1")); 89 | context.assertEquals(42L, result.get("key1")); 90 | context.assertNotNull(result.get("key2")); 91 | context.assertEquals(4711L, result.get("key2")); 92 | async.complete(); 93 | 94 | future.complete(); 95 | }, context.asyncAssertSuccess()); 96 | 97 | })); 98 | 99 | } 100 | 101 | @Test 102 | public void notFound(TestContext context) { 103 | Async async = context.async(); 104 | 105 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 106 | 107 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 108 | 109 | msg.fail(404, "does not matter"); 110 | 111 | }); 112 | 113 | rule.vertx().executeBlocking(future -> { 114 | 115 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 116 | context.assertTrue(result.isEmpty()); 117 | async.complete(); 118 | 119 | future.complete(); 120 | }, context.asyncAssertSuccess()); 121 | 122 | })); 123 | 124 | } 125 | 126 | @Test 127 | public void badRequest(TestContext context) { 128 | Async async = context.async(); 129 | 130 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 131 | 132 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 133 | 134 | msg.fail(400, "does not matter"); 135 | 136 | }); 137 | 138 | rule.vertx().executeBlocking(future -> { 139 | 140 | 141 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 142 | context.fail(); 143 | 144 | 145 | }, context.asyncAssertFailure(ex -> { 146 | 147 | context.assertTrue(ex instanceof IllegalArgumentException); 148 | async.complete(); 149 | })); 150 | 151 | })); 152 | 153 | } 154 | 155 | @Test 156 | public void internalServerError(TestContext context) { 157 | Async async = context.async(); 158 | 159 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 160 | 161 | rule.vertx().eventBus().consumer(Config.ALL_KEY_VALUE_QUERY_FACADE_ADDRESS, msg -> { 162 | 163 | msg.fail(500, "does not matter"); 164 | 165 | }); 166 | 167 | rule.vertx().executeBlocking(future -> { 168 | 169 | 170 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 171 | context.fail(); 172 | 173 | 174 | }, context.asyncAssertFailure(ex -> { 175 | context.assertTrue(ex instanceof QueryExecutionException); 176 | async.complete(); 177 | })); 178 | 179 | })); 180 | 181 | } 182 | 183 | @Test 184 | public void connectionExceptionInvalidPort(TestContext context) { 185 | 186 | 187 | Async async = context.async(); 188 | 189 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 190 | 191 | rule.vertx().executeBlocking(future -> { 192 | 193 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("localhost", -1); 194 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 195 | 196 | context.fail(); 197 | 198 | 199 | }, context.asyncAssertFailure(ex -> { 200 | context.assertTrue(ex instanceof ConnectionException); 201 | async.complete(); 202 | })); 203 | 204 | })); 205 | 206 | } 207 | 208 | @Test 209 | public void connectionExceptionInvalidHost(TestContext context) { 210 | Async async = context.async(); 211 | 212 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 213 | 214 | rule.vertx().executeBlocking(future -> { 215 | 216 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("host with spaces", 4567); 217 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 218 | 219 | context.fail(); 220 | 221 | 222 | }, context.asyncAssertFailure(ex -> { 223 | context.assertTrue(ex instanceof ConnectionException); 224 | async.complete(); 225 | })); 226 | 227 | })); 228 | 229 | } 230 | 231 | @Test 232 | public void validButUnReachableHost(TestContext context) { 233 | Async async = context.async(); 234 | 235 | rule.vertx().deployVerticle(new MockedRuntimeHttpServerVerticle(new HttpServerOptions().setPort(4567), new DummyVerticle()), context.asyncAssertSuccess(handler -> { 236 | 237 | rule.vertx().executeBlocking(future -> { 238 | 239 | unitUnderTest = new GenericBlockingRestKiqrClientImpl("ihopethisdoesntexist", 4567); 240 | Map result = unitUnderTest.getAllKeyValues("store", String.class, Long.class, Serdes.String(), Serdes.Long()); 241 | context.fail(); 242 | 243 | 244 | }, context.asyncAssertFailure(ex -> { 245 | context.assertTrue(ex instanceof ConnectionException); 246 | async.complete(); 247 | })); 248 | 249 | })); 250 | 251 | } 252 | 253 | } 254 | --------------------------------------------------------------------------------