├── .gitignore
├── src
    ├── main
    │   ├── resources
    │   │   └── log4j.xml
    │   ├── config
    │   │   ├── luwak.properties
    │   │   └── combiner.properties
    │   ├── java
    │   │   └── uk
    │   │   │   └── co
    │   │   │       └── flax
    │   │   │           └── samzaluwak
    │   │   │               ├── Interact.java
    │   │   │               ├── MatchRecombinerTask.java
    │   │   │               └── MonitorTask.java
    │   └── assembly
    │   │   └── src.xml
    └── test
    │   └── java
    │       └── uk
    │           └── co
    │               └── flax
    │                   └── samzaluwak
    │                       └── TestMonitorTask.java
├── README.md
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | target
2 | deploy
3 | *.iml
4 | 
5 | .idea/
6 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" ?>
 2 | <!--
 3 | 
 4 |  Licensed to the Apache Software Foundation (ASF) under one
 5 |  or more contributor license agreements.  See the NOTICE file
 6 |  distributed with this work for additional information
 7 |  regarding copyright ownership.  The ASF licenses this file
 8 |  to you under the Apache License, Version 2.0 (the
 9 |  "License"); you may not use this file except in compliance
10 |  with the License.  You may obtain a copy of the License at
11 | 
12 |    http://www.apache.org/licenses/LICENSE-2.0
13 | 
14 |  Unless required by applicable law or agreed to in writing,
15 |  software distributed under the License is distributed on an
16 |  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 |  KIND, either express or implied.  See the License for the
18 |  specific language governing permissions and limitations
19 |  under the License.
20 | 
21 | -->
22 | 
23 | <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
24 | <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
25 |   <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
26 | 
27 |   <appender name="RollingAppender" class="org.apache.log4j.DailyRollingFileAppender">
28 |      <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
29 |      <param name="DatePattern" value="'.'yyyy-MM-dd" />
30 |      <layout class="org.apache.log4j.PatternLayout">
31 |       <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
32 |      </layout>
33 |   </appender>
34 |   <root>
35 |     <priority value="info" />
36 |     <appender-ref ref="RollingAppender"/>
37 |     <appender-ref ref="jmx" />
38 |   </root>
39 | </log4j:configuration>
40 | 


--------------------------------------------------------------------------------
/src/main/config/luwak.properties:
--------------------------------------------------------------------------------
 1 | # Licensed to the Apache Software Foundation (ASF) under one
 2 | # or more contributor license agreements.  See the NOTICE file
 3 | # distributed with this work for additional information
 4 | # regarding copyright ownership.  The ASF licenses this file
 5 | # to you under the Apache License, Version 2.0 (the
 6 | # "License"); you may not use this file except in compliance
 7 | # with the License.  You may obtain a copy of the License at
 8 | #
 9 | #   http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing,
12 | # software distributed under the License is distributed on an
13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | # KIND, either express or implied.  See the License for the
15 | # specific language governing permissions and limitations
16 | # under the License.
17 | 
18 | # Job
19 | job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
20 | job.name=luwak
21 | 
22 | # YARN
23 | yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-dist.tar.gz
24 | 
25 | # Task
26 | task.class=uk.co.flax.samzaluwak.MonitorTask
27 | task.inputs=kafka.queries,kafka.documents
28 | task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
29 | task.checkpoint.system=kafka
30 | task.checkpoint.replication.factor=1
31 | task.consumer.batch.size=1
32 | 
33 | # Serializers
34 | serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
35 | serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
36 | 
37 | # Kafka System
38 | systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
39 | systems.kafka.samza.msg.serde=json
40 | systems.kafka.samza.key.serde=string
41 | systems.kafka.consumer.zookeeper.connect=localhost:2181/
42 | systems.kafka.producer.metadata.broker.list=localhost:9092
43 | systems.kafka.producer.producer.type=sync
44 | # Normally, we'd set this much higher, but we want things to look snappy in the demo.
45 | systems.kafka.producer.batch.num.messages=1
46 | 
47 | systems.kafka.streams.queries.samza.bootstrap=true
48 | systems.kafka.streams.queries.samza.reset.offset=true
49 | systems.kafka.streams.queries.samza.offset.default=oldest
50 | 


--------------------------------------------------------------------------------
/src/main/config/combiner.properties:
--------------------------------------------------------------------------------
 1 | # Licensed to the Apache Software Foundation (ASF) under one
 2 | # or more contributor license agreements.  See the NOTICE file
 3 | # distributed with this work for additional information
 4 | # regarding copyright ownership.  The ASF licenses this file
 5 | # to you under the Apache License, Version 2.0 (the
 6 | # "License"); you may not use this file except in compliance
 7 | # with the License.  You may obtain a copy of the License at
 8 | #
 9 | #   http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing,
12 | # software distributed under the License is distributed on an
13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | # KIND, either express or implied.  See the License for the
15 | # specific language governing permissions and limitations
16 | # under the License.
17 | 
18 | # Job
19 | job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
20 | job.name=combiner
21 | 
22 | # YARN
23 | yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-dist.tar.gz
24 | 
25 | # Task
26 | task.class=uk.co.flax.samzaluwak.MatchRecombinerTask
27 | task.inputs=kafka.matches1
28 | task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
29 | task.checkpoint.system=kafka
30 | task.checkpoint.replication.factor=1
31 | task.consumer.batch.size=1
32 | 
33 | # Serializers
34 | serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
35 | serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
36 | 
37 | # Kafka System
38 | systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
39 | systems.kafka.samza.msg.serde=json
40 | systems.kafka.samza.key.serde=string
41 | systems.kafka.consumer.zookeeper.connect=localhost:2181/
42 | systems.kafka.producer.metadata.broker.list=localhost:9092
43 | systems.kafka.producer.producer.type=sync
44 | # Normally, we'd set this much higher, but we want things to look snappy in the demo.
45 | systems.kafka.producer.batch.num.messages=1
46 | 
47 | stores.matches.factory=org.apache.samza.storage.kv.LevelDbKeyValueStorageEngineFactory
48 | stores.matches.changelog=kafka.matches-combiner-changelog
49 | stores.matches.key.serde=string
50 | stores.matches.msg.serde=json


--------------------------------------------------------------------------------
/src/test/java/uk/co/flax/samzaluwak/TestMonitorTask.java:
--------------------------------------------------------------------------------
 1 | package uk.co.flax.samzaluwak;
 2 | 
 3 | import com.google.common.collect.ImmutableMap;
 4 | import org.apache.samza.Partition;
 5 | import org.apache.samza.system.IncomingMessageEnvelope;
 6 | import org.apache.samza.system.OutgoingMessageEnvelope;
 7 | import org.apache.samza.system.SystemStreamPartition;
 8 | import org.apache.samza.task.MessageCollector;
 9 | import org.junit.Test;
10 | 
11 | import static org.mockito.Matchers.any;
12 | import static org.mockito.Mockito.mock;
13 | import static org.mockito.Mockito.times;
14 | import static org.mockito.Mockito.verify;
15 | 
16 | /**
17 |  * Copyright (c) 2014 Lemur Consulting Ltd.
18 |  * <p/>
19 |  * Licensed under the Apache License, Version 2.0 (the "License");
20 |  * you may not use this file except in compliance with the License.
21 |  * You may obtain a copy of the License at
22 |  * <p/>
23 |  * http://www.apache.org/licenses/LICENSE-2.0
24 |  * <p/>
25 |  * Unless required by applicable law or agreed to in writing, software
26 |  * distributed under the License is distributed on an "AS IS" BASIS,
27 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
28 |  * See the License for the specific language governing permissions and
29 |  * limitations under the License.
30 |  */
31 | 
32 | public class TestMonitorTask {
33 | 
34 |     public static final SystemStreamPartition QUERY_PART = new SystemStreamPartition("kafka", MonitorTask.QUERIES_STREAM, new Partition(0));
35 |     public static final SystemStreamPartition DOCS_PART = new SystemStreamPartition("kafka", MonitorTask.DOCS_STREAM, new Partition(0));
36 | 
37 |     @Test
38 |     public void testTask() throws Exception {
39 | 
40 |         MonitorTask task = new MonitorTask();
41 |         task.init(null, null);
42 | 
43 |         MessageCollector collector = mock(MessageCollector.class);
44 | 
45 |         IncomingMessageEnvelope query = new IncomingMessageEnvelope(QUERY_PART, "", "1", ImmutableMap.of("query", "hello world"));
46 |         task.process(query, collector, null);
47 | 
48 |         IncomingMessageEnvelope doc = new IncomingMessageEnvelope(DOCS_PART, "", "doc1", ImmutableMap.of("f", "hello world"));
49 |         task.process(doc, collector, null);
50 | 
51 |         verify(collector, times(1)).send(any(OutgoingMessageEnvelope.class));
52 | 
53 |     }
54 | 
55 | 
56 | }
57 | 


--------------------------------------------------------------------------------
/src/main/java/uk/co/flax/samzaluwak/Interact.java:
--------------------------------------------------------------------------------
 1 | package uk.co.flax.samzaluwak;
 2 | 
 3 | import java.io.BufferedReader;
 4 | import java.io.IOException;
 5 | import java.io.InputStreamReader;
 6 | import java.util.Properties;
 7 | 
 8 | import kafka.javaapi.producer.Producer;
 9 | import kafka.producer.KeyedMessage;
10 | import kafka.producer.ProducerConfig;
11 | 
12 | /**
13 |  * Copyright (c) 2014 Lemur Consulting Ltd.
14 |  * <p/>
15 |  * Licensed under the Apache License, Version 2.0 (the "License");
16 |  * you may not use this file except in compliance with the License.
17 |  * You may obtain a copy of the License at
18 |  * <p/>
19 |  * http://www.apache.org/licenses/LICENSE-2.0
20 |  * <p/>
21 |  * Unless required by applicable law or agreed to in writing, software
22 |  * distributed under the License is distributed on an "AS IS" BASIS,
23 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24 |  * See the License for the specific language governing permissions and
25 |  * limitations under the License.
26 |  */
27 | 
28 | public class Interact {
29 | 
30 |     public static void main(String... args) throws IOException {
31 | 
32 |         Properties props = new Properties();
33 |         props.setProperty("metadata.broker.list", "localhost:9092");
34 |         props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
35 |         ProducerConfig config = new ProducerConfig(props);
36 | 
37 |         Producer<String, String> producer = new Producer<>(config);
38 | 
39 |         BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
40 | 
41 |         while (true) {
42 |             String cmd = reader.readLine();
43 |             if ("quit".equals(cmd))
44 |                 return;
45 |             if (cmd.startsWith("q "))
46 |                 sendQueryUpdate(cmd, producer);
47 |             if (cmd.startsWith("d "))
48 |                 sendDocument(cmd, producer);
49 |         }
50 | 
51 |     }
52 | 
53 |     private static void sendQueryUpdate(String cmd, Producer<String, String> producer) {
54 | 
55 |         String[] parts = cmd.split("\\s", 3);
56 |         String id = parts[1];
57 |         String query = parts[2];
58 | 
59 |         KeyedMessage<String, String> message
60 |                 = new KeyedMessage<>(MonitorTask.QUERIES_STREAM, id, "{ \"query\" : \"" + query + "\"}");
61 |         producer.send(message);
62 | 
63 |     }
64 | 
65 |     private static void sendDocument(String cmd, Producer<String, String> producer) {
66 | 
67 |         String[] parts = cmd.split("\\s", 3);
68 |         String id = parts[1];
69 |         String doc = parts[2];
70 | 
71 |         for (int i = 0; i < MatchRecombinerTask.QUERY_PARTITIONS; i++) {
72 |             KeyedMessage<String, String> message =
73 |                     new KeyedMessage<>(MonitorTask.DOCS_STREAM, id, i, "{ \"f\" : \"" + doc + "\"}");
74 |             producer.send(message);
75 |         }
76 | 
77 |     }
78 | 
79 | }
80 | 


--------------------------------------------------------------------------------
/src/main/assembly/src.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
 3 |   license agreements. See the NOTICE file distributed with this work for additional
 4 |   information regarding copyright ownership. The ASF licenses this file to
 5 |   you under the Apache License, Version 2.0 (the "License"); you may not use
 6 |   this file except in compliance with the License. You may obtain a copy of
 7 |   the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
 8 |   by applicable law or agreed to in writing, software distributed under the
 9 |   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
10 |   OF ANY KIND, either express or implied. See the License for the specific
11 |   language governing permissions and limitations under the License. -->
12 | 
13 | <assembly
14 |   xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
15 |   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
16 |   xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
17 |   <id>dist</id>
18 |   <formats>
19 |     <format>tar.gz</format>
20 |   </formats>
21 |   <includeBaseDirectory>false</includeBaseDirectory>
22 |   <files>
23 |     <file>
24 |       <source>${basedir}/src/main/resources/log4j.xml</source>
25 |       <outputDirectory>lib</outputDirectory>
26 |     </file>
27 |     <!-- filtered=true, so we do variable expansion so the yarn package path
28 |       always points to the correct spot on any machine -->
29 |     <file>
30 |       <source>${basedir}/src/main/config/luwak.properties</source>
31 |       <outputDirectory>config</outputDirectory>
32 |       <filtered>true</filtered>
33 |     </file>
34 |       <file>
35 |           <source>${basedir}/src/main/config/combiner.properties</source>
36 |           <outputDirectory>config</outputDirectory>
37 |           <filtered>true</filtered>
38 |       </file>
39 |   </files>
40 |   <dependencySets>
41 |     <dependencySet>
42 |       <outputDirectory>bin</outputDirectory>
43 |       <includes>
44 |         <include>org.apache.samza:samza-shell:tgz:dist:*</include>
45 |       </includes>
46 |       <fileMode>0744</fileMode>
47 |       <unpack>true</unpack>
48 |     </dependencySet>
49 |     <dependencySet>
50 |       <outputDirectory>lib</outputDirectory>
51 |       <includes>
52 |         <include>org.apache.samza:samza-core_2.10</include>
53 |         <include>org.apache.samza:samza-kafka_2.10</include>
54 |         <include>org.apache.samza:samza-serializers_2.10</include>
55 |         <include>org.apache.samza:samza-yarn_2.10</include>
56 |         <include>org.apache.samza:samza-kv-leveldb_2.10</include>
57 |         <include>org.apache.samza:samza-log4j</include>
58 |         <include>org.slf4j:slf4j-log4j12</include>
59 |         <include>org.apache.kafka:kafka_2.10</include>
60 |         <include>org.apache.hadoop:hadoop-hdfs</include>
61 |         <include>uk.co.flax:samza-luwak</include>
62 |         <include>uk.co.flax:luwak</include>
63 |       </includes>
64 |       <useTransitiveFiltering>true</useTransitiveFiltering>
65 |     </dependencySet>
66 |   </dependencySets>
67 | </assembly>
68 | 


--------------------------------------------------------------------------------
/src/main/java/uk/co/flax/samzaluwak/MatchRecombinerTask.java:
--------------------------------------------------------------------------------
 1 | package uk.co.flax.samzaluwak;
 2 | 
 3 | import java.util.HashMap;
 4 | import java.util.List;
 5 | import java.util.Map;
 6 | 
 7 | import com.google.common.collect.Lists;
 8 | import org.apache.samza.config.Config;
 9 | import org.apache.samza.storage.kv.Entry;
10 | import org.apache.samza.storage.kv.KeyValueIterator;
11 | import org.apache.samza.storage.kv.KeyValueStore;
12 | import org.apache.samza.system.IncomingMessageEnvelope;
13 | import org.apache.samza.system.OutgoingMessageEnvelope;
14 | import org.apache.samza.system.SystemStream;
15 | import org.apache.samza.task.*;
16 | import org.slf4j.Logger;
17 | import org.slf4j.LoggerFactory;
18 | 
19 | /**
20 |  * Copyright (c) 2014 Lemur Consulting Ltd.
21 |  * <p/>
22 |  * Licensed under the Apache License, Version 2.0 (the "License");
23 |  * you may not use this file except in compliance with the License.
24 |  * You may obtain a copy of the License at
25 |  * <p/>
26 |  * http://www.apache.org/licenses/LICENSE-2.0
27 |  * <p/>
28 |  * Unless required by applicable law or agreed to in writing, software
29 |  * distributed under the License is distributed on an "AS IS" BASIS,
30 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
31 |  * See the License for the specific language governing permissions and
32 |  * limitations under the License.
33 |  */
34 | 
35 | public class MatchRecombinerTask implements StreamTask, InitableTask {
36 | 
37 |     private static final Logger logger = LoggerFactory.getLogger(MatchRecombinerTask.class);
38 | 
39 |     public static final int QUERY_PARTITIONS = 2;
40 | 
41 |     private KeyValueStore<String, Map<String, String>> store;
42 | 
43 |     public static final SystemStream MATCHES_STREAM = new SystemStream("kafka", "combinedmatches");
44 | 
45 |     @Override
46 |     public void process(IncomingMessageEnvelope message, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
47 |         String key = (String) message.getKey();
48 |         Map<String, String> matches = (Map<String, String>) message.getMessage();
49 |         store.put(key, matches);
50 |         logger.info("Got partial match for {}", key);
51 | 
52 |         String originalKey = originalKey(key);
53 | 
54 |         Map<String, Map<String, String>> parts = collectMatches(originalKey);
55 |         if (parts.size() != QUERY_PARTITIONS)
56 |             return;
57 | 
58 |         logger.info("All partial matches for {} received", originalKey);
59 |         List<Map<String, String>> combinedMatches = combineMatches(parts);
60 | 
61 |         collector.send(new OutgoingMessageEnvelope(MATCHES_STREAM, originalKey, combinedMatches));
62 |         for (String storekey : parts.keySet()) {
63 |             store.delete(storekey);
64 |         }
65 | 
66 |     }
67 | 
68 |     private List<Map<String, String>> combineMatches(Map<String, Map<String, String>> parts) {
69 |         return Lists.newArrayList(parts.values());
70 |     }
71 | 
72 |     @Override
73 |     public void init(Config config, TaskContext taskContext) throws Exception {
74 |         this.store = (KeyValueStore<String, Map<String, String>>) taskContext.getStore("matches");
75 |     }
76 | 
77 |     private Map<String, Map<String, String>> collectMatches(String key) {
78 |         Map<String, Map<String, String>> parts = new HashMap<>();
79 |         KeyValueIterator<String, Map<String, String>> it = store.range(key + "_", key + "_a");
80 |         try {
81 |             while (it.hasNext()) {
82 |                 Entry<String, Map<String, String>> entry = it.next();
83 |                 parts.put(entry.getKey(), entry.getValue());
84 |             }
85 |             logger.info("Found {} partial matches in store for {}", parts.size(), key);
86 |             return parts;
87 |         }
88 |         finally {
89 |             it.close();
90 |         }
91 | 
92 |     }
93 | 
94 |     private String originalKey(String key) {
95 |         return key.replaceAll("_.*?$", "");
96 |     }
97 | }
98 | 


--------------------------------------------------------------------------------
/src/main/java/uk/co/flax/samzaluwak/MonitorTask.java:
--------------------------------------------------------------------------------
 1 | package uk.co.flax.samzaluwak;
 2 | 
 3 | import java.io.IOException;
 4 | import java.util.Map;
 5 | 
 6 | import org.apache.lucene.analysis.Analyzer;
 7 | import org.apache.lucene.analysis.standard.StandardAnalyzer;
 8 | import org.apache.samza.config.Config;
 9 | import org.apache.samza.system.IncomingMessageEnvelope;
10 | import org.apache.samza.system.OutgoingMessageEnvelope;
11 | import org.apache.samza.system.SystemStream;
12 | import org.apache.samza.task.*;
13 | import org.slf4j.Logger;
14 | import org.slf4j.LoggerFactory;
15 | import uk.co.flax.luwak.*;
16 | import uk.co.flax.luwak.matchers.SimpleMatcher;
17 | import uk.co.flax.luwak.presearcher.MatchAllPresearcher;
18 | import uk.co.flax.luwak.queryparsers.LuceneQueryParser;
19 | 
20 | /**
21 |  * Copyright (c) 2014 Lemur Consulting Ltd.
22 |  * <p/>
23 |  * Licensed under the Apache License, Version 2.0 (the "License");
24 |  * you may not use this file except in compliance with the License.
25 |  * You may obtain a copy of the License at
26 |  * <p/>
27 |  * http://www.apache.org/licenses/LICENSE-2.0
28 |  * <p/>
29 |  * Unless required by applicable law or agreed to in writing, software
30 |  * distributed under the License is distributed on an "AS IS" BASIS,
31 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
32 |  * See the License for the specific language governing permissions and
33 |  * limitations under the License.
34 |  */
35 | 
36 | public class MonitorTask implements StreamTask, InitableTask {
37 | 
38 |     private static final Logger logger = LoggerFactory.getLogger(MonitorTask.class);
39 | 
40 |     public static final String QUERIES_STREAM = "queries";
41 |     public static final String DOCS_STREAM = "documents";
42 | 
43 |     private Monitor monitor;
44 |     private Analyzer analyzer = new StandardAnalyzer();
45 |     private MatcherFactory<QueryMatch> matcherFactory = SimpleMatcher.FACTORY;
46 | 
47 |     public static final SystemStream MATCHES_STREAM = new SystemStream("kafka", "matches1");
48 | 
49 |     @Override
50 |     public void process(IncomingMessageEnvelope message, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
51 |         String stream = message.getSystemStreamPartition().getStream();
52 |         switch (stream) {
53 |             case QUERIES_STREAM:
54 |                 logger.info("Adding new query for partition {}: {}", message.getSystemStreamPartition().getPartition().getPartitionId(), message.getMessage());
55 |                 update((String) message.getKey(), (Map<String, String>) message.getMessage());
56 |                 break;
57 |             case DOCS_STREAM:
58 |                 Matches<QueryMatch> matches = match((String) message.getKey(), (Map<String, String>) message.getMessage());
59 |                 String key = message.getKey() + "_" + message.getSystemStreamPartition().getPartition().getPartitionId();
60 |                 collector.send(new OutgoingMessageEnvelope(MATCHES_STREAM, message.getKey(), key, matches));
61 |                 break;
62 |             default:
63 |                 throw new RuntimeException("Unknown stream: " + stream);
64 |         }
65 |     }
66 | 
67 |     @Override
68 |     public void init(Config config, TaskContext taskContext) throws Exception {
69 |         monitor = new Monitor(new LuceneQueryParser("f"), new MatchAllPresearcher());
70 |     }
71 | 
72 |     private void update(String id, Map<String, String> message) throws IOException {
73 |         MonitorQuery mq = new MonitorQuery(id, message.get("query"), message);
74 |         logger.info("Adding new query: {}", mq);
75 |         for (QueryError error : monitor.update(mq)) {
76 |             logger.warn(error.toString());
77 |         }
78 |     }
79 | 
80 |     private Matches<QueryMatch> match(String id, Map<String, String> fields) throws IOException {
81 | 
82 |         InputDocument.Builder builder = InputDocument.builder(id);
83 |         for (Map.Entry<String, String> field : fields.entrySet()) {
84 |             builder.addField(field.getKey(), field.getValue(), analyzer);
85 |         }
86 | 
87 |         logger.info("Matching document {}", id);
88 |         return monitor.match(builder.build(), matcherFactory);
89 | 
90 |     }
91 | 
92 | }
93 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Samza-Luwak Proof of Concept
  2 | ============================
  3 | 
  4 | This project is an experimental setup for using [Luwak](https://github.com/flaxsearch/luwak)
  5 | within [Samza](http://samza.incubator.apache.org/).
  6 | 
  7 | Use case: imagine you want to implement something like
  8 | [Google Alerts](https://www.google.com/alerts). Every time a new document is published, you want to
  9 | run it through a list of search queries, and notify someone if the document matches one of the
 10 | queries. (There's actually a [whole industry](http://en.wikipedia.org/wiki/Media_monitoring) that
 11 | specialises in doing this.) A similar use case is Twitter search, when you want to see a stream of
 12 | tweets matching a search query.
 13 | 
 14 | We're exploring ways of making such streaming search queries scalable (scaling both to large
 15 | throughput of documents and large numbers of queries), using the following tools:
 16 | 
 17 | * [Luwak](https://github.com/flaxsearch/luwak) is a wrapper library around
 18 |   [Lucene](http://lucene.apache.org/). Lucene does the document indexing and querying, and Luwak
 19 |   adds some optimisations to improve performance when each document needs to be matched against
 20 |   a large number (hundreds of thousands) of queries.
 21 | * [Samza](http://samza.incubator.apache.org/) is a distributed stream processing framework
 22 |   based on [Kafka](http://kafka.apache.org/) and
 23 |   [Hadoop YARN](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html).
 24 |   It provides fault-tolerant message processing and a cluster deployment mechanism.
 25 | 
 26 | What we're doing is a bit similar to
 27 | [ElasticSearch's percolator](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-percolate.html),
 28 | but we think it has the potential to perform better and be more scalable.
 29 | 
 30 | 
 31 | High-level architecture
 32 | -----------------------
 33 | 
 34 | There are two input streams and one output stream, which are implemented as Kafka topics:
 35 | 
 36 | * **Queries** (input). When a user wants to start being alerted about documents matching some
 37 |   search query, they send a message to the queries stream. A unique ID is used as the key of the
 38 |   message. When the user wants to modify a query, or stop alerts for a particular query, they can
 39 |   update it by sending another message with the same key.
 40 | * **Documents** (input). Every document is published to a stream, and Luwak matches each document
 41 |   against the current set of stored queries.
 42 | * **Matches** (output). For each document that comes in, the job produces an output message that
 43 |   lists the IDs of the queries that matched the document. This can then be used by downstream
 44 |   consumers to alert the owners of those queries.
 45 | 
 46 | Note that there is currently no persistent index of documents. This is pure stream search: search
 47 | results include only documents that are published after the query is registered, but not historical
 48 | documents.
 49 | 
 50 | 
 51 | Partitioning
 52 | ------------
 53 | 
 54 | How do we make this fast and scalable? Evaluating a single search query is fast, but if you're
 55 | matching each document against hundreds of thousands of queries (some of which can be very complex),
 56 | it can take a significant time to process every document. We are investigating several approaches:
 57 | 
 58 | * **Co-partitioning queries and documents**. In some cases, you know in advance that some queries 
 59 |   can only apply to certain documents, for example English-language queries are only applied to
 60 |   English-language documents. This creates a natural partitioning scheme, but requires that you
 61 |   define the partitioning manually.
 62 | * **Pre-searching**. If a query requires an unusual word to appear in the document, then that
 63 |   query only needs to be attempted on documents containing that word. Luwak already does this to
 64 |   some degree, and perhaps we can take it further.
 65 | * **Partitioning the query set**. We can have several partitions of the stream processor, and store
 66 |   each query on one of the partitions. Then each partition has a smaller number of queries to try,
 67 |   so each document can be matched faster. However, each document must be sent to all partitions of
 68 |   the stream processor, unless we have some prior knowledge about which queries may apply (using
 69 |   pre-searching).
 70 | * **Partitioning the document stream**. We can keep the entire list of queries on each stream
 71 |   processing node, and parallelise the processing by sending each document to only one of the
 72 |   nodes. This does not make the processing of any individual document any faster, but allows the
 73 |   process to scale to a higher throughput of documents.
 74 | 
 75 | At the moment, we are exploring partitioning of the query set. This requires a multi-stage pipeline:
 76 | 
 77 | 1. Documents need to be published to *all* partitions of the documents stream.
 78 |    Queries need to be partitioned by query ID (and only published to one partition).
 79 | 2. The matching job matches each documents against the queries within its partition, and emits a
 80 |    message to the "matches" stream, indicating which query IDs matched that document ID.
 81 | 3. A second job consumes the "matches" stream, and waits for the matches from all the partitions
 82 |    to arrive. When all partitions have processed a particular document, this job emits a message
 83 |    to the "combined-matches" stream, which now includes the matching query IDs from all
 84 |    partitions.
 85 | 
 86 | 
 87 | Building
 88 | --------
 89 | 
 90 | This project is very hacky and experimental, and may not work at all. It's also a bit convoluted
 91 | to build right now, because it depends on various unreleased components.
 92 | 
 93 | Check out [Samza](https://github.com/apache/incubator-samza) (master branch), build it and install
 94 | it to your local Maven repository:
 95 | 
 96 | ```bash
 97 | git clone https://github.com/apache/incubator-samza.git samza
 98 | cd samza
 99 | ./gradlew -PscalaVersion=2.10 clean publishToMavenLocal
100 | ```
101 | 
102 | Check out [hello-samza](http://samza.incubator.apache.org/startup/hello-samza/latest/index.html)
103 | (latest branch), and use it to launch Zookeeper, Kafka and YARN locally:
104 | 
105 | ```bash
106 | git clone https://github.com/apache/incubator-samza-hello-samza.git hello-samza
107 | cd hello-samza
108 | git checkout latest
109 | bin/grid bootstrap
110 | bin/grid start all
111 | for topic in documents queries matches1 combinedmatches matches-combiner-changelog; do
112 |   deploy/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic $topic --partitions 2 --replication-factor 1
113 | done
114 | ```
115 | 
116 | Check out [Flax](http://www.flax.co.uk/)'s
117 | [fork of Lucene](https://github.com/flaxsearch/lucene-solr-intervals) (positions-5x branch),
118 | which has a new feature that Luwak needs, but is not yet upstream
119 | (it should be committed upstream eventually):
120 | 
121 | ```bash
122 | git clone https://github.com/flaxsearch/lucene-solr-intervals.git
123 | cd lucene-solr-internals
124 | git checkout positions-5x
125 | mvn -DskipTests install
126 | ```
127 | 
128 | Check out [Luwak](https://github.com/flaxsearch/luwak) (1.1.x branch), build it and install it
129 | to your local Maven repository (note this currently doesn't work with JDK8):
130 | 
131 | ```bash
132 | git clone https://github.com/flaxsearch/luwak.git
133 | cd luwak
134 | git checkout 1.1.x
135 | mvn install
136 | ```
137 | 
138 | Build and run this project:
139 | 
140 | ```bash
141 | git clone https://github.com/romseygeek/samza-luwak.git
142 | cd samza-luwak
143 | mvn clean package
144 | mkdir deploy
145 | tar -xzvf target/samza-luwak-1.0-SNAPSHOT-dist.tar.gz -C deploy/
146 | deploy/bin/run-job.sh --config-path=file://$PWD/deploy/config/luwak.properties
147 | deploy/bin/run-job.sh --config-path=file://$PWD/deploy/config/combiner.properties
148 | ```
149 | 
150 | Now you can try adding some test documents and queries to the system, and observe the output:
151 | 
152 | ```bash
153 | hello-samza/deploy/kafka/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic combinedmatches --from-beginning &
154 | java -jar samza-luwak/target/samza-luwak-1.0-SNAPSHOT.jar
155 | q query1 foo AND bar
156 | q query2 bar AND baz
157 | d doc1 this document contains the words foo and bar only
158 | d doc2 this document, on the other hand, mentions bar and baz.
159 | d doc3 this one goes nuts and mentions foo, bar and baz -- all three!
160 | d doc4 finally, this one mentions none of those words.
161 | quit
162 | ```
163 | 
164 | In that command-line tool, queries are defined with "q", followed by the query ID, followed by a
165 | Lucene query string. Documents are defined with "d", followed by the document ID, followed by the
166 | text of the document.
167 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |     <modelVersion>4.0.0</modelVersion>
  6 | 
  7 |     <groupId>uk.co.flax</groupId>
  8 |     <artifactId>samza-luwak</artifactId>
  9 |     <version>1.0-SNAPSHOT</version>
 10 | 
 11 |     <dependencies>
 12 |         <dependency>
 13 |             <groupId>org.apache.samza</groupId>
 14 |             <artifactId>samza-api</artifactId>
 15 |             <version>${samza.version}</version>
 16 |         </dependency>
 17 |         <dependency>
 18 |             <groupId>org.apache.samza</groupId>
 19 |             <artifactId>samza-core_2.10</artifactId>
 20 |             <version>${samza.version}</version>
 21 |         </dependency>
 22 |         <dependency>
 23 |             <groupId>org.apache.samza</groupId>
 24 |             <artifactId>samza-log4j</artifactId>
 25 |             <version>${samza.version}</version>
 26 |         </dependency>
 27 |         <dependency>
 28 |             <groupId>org.apache.samza</groupId>
 29 |             <artifactId>samza-serializers_2.10</artifactId>
 30 |             <version>${samza.version}</version>
 31 |         </dependency>
 32 |         <dependency>
 33 |             <groupId>org.apache.samza</groupId>
 34 |             <artifactId>samza-shell</artifactId>
 35 |             <classifier>dist</classifier>
 36 |             <type>tgz</type>
 37 |             <version>${samza.version}</version>
 38 |         </dependency>
 39 |         <dependency>
 40 |             <groupId>org.apache.samza</groupId>
 41 |             <artifactId>samza-yarn_2.10</artifactId>
 42 |             <version>${samza.version}</version>
 43 |         </dependency>
 44 |         <dependency>
 45 |             <groupId>org.apache.samza</groupId>
 46 |             <artifactId>samza-kv_2.10</artifactId>
 47 |             <version>${samza.version}</version>
 48 |         </dependency>
 49 |         <dependency>
 50 |             <groupId>org.apache.samza</groupId>
 51 |             <artifactId>samza-kv-leveldb_2.10</artifactId>
 52 |             <version>${samza.version}</version>
 53 |         </dependency>
 54 |         <dependency>
 55 |             <groupId>org.apache.samza</groupId>
 56 |             <artifactId>samza-kafka_2.10</artifactId>
 57 |             <version>${samza.version}</version>
 58 |         </dependency>
 59 |         <dependency>
 60 |             <groupId>org.apache.kafka</groupId>
 61 |             <artifactId>kafka_2.10</artifactId>
 62 |             <version>0.8.1</version>
 63 |         </dependency>
 64 |         <dependency>
 65 |             <groupId>org.schwering</groupId>
 66 |             <artifactId>irclib</artifactId>
 67 |             <version>1.10</version>
 68 |         </dependency>
 69 |         <dependency>
 70 |             <groupId>org.slf4j</groupId>
 71 |             <artifactId>slf4j-api</artifactId>
 72 |             <version>1.6.2</version>
 73 |         </dependency>
 74 |         <dependency>
 75 |             <groupId>org.slf4j</groupId>
 76 |             <artifactId>slf4j-log4j12</artifactId>
 77 |             <version>1.6.2</version>
 78 |         </dependency>
 79 |         <dependency>
 80 |             <groupId>org.codehaus.jackson</groupId>
 81 |             <artifactId>jackson-jaxrs</artifactId>
 82 |             <version>1.8.5</version>
 83 |         </dependency>
 84 |         <dependency>
 85 |             <groupId>org.apache.hadoop</groupId>
 86 |             <artifactId>hadoop-hdfs</artifactId>
 87 |             <version>2.4.0</version>
 88 |         </dependency>
 89 |         <dependency>
 90 |             <groupId>uk.co.flax</groupId>
 91 |             <artifactId>luwak</artifactId>
 92 |             <version>1.1.0-rc4-SNAPSHOT</version>
 93 |         </dependency>
 94 |         <dependency>
 95 |             <groupId>junit</groupId>
 96 |             <artifactId>junit</artifactId>
 97 |             <version>4.11</version>
 98 |             <scope>test</scope>
 99 |         </dependency>
100 |         <dependency>
101 |             <groupId>org.mockito</groupId>
102 |             <artifactId>mockito-core</artifactId>
103 |             <version>1.9.5</version>
104 |             <scope>test</scope>
105 |         </dependency>
106 |     </dependencies>
107 | 
108 |     <properties>
109 |         <!-- maven specific properties -->
110 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
111 |         <samza.version>0.8.0-SNAPSHOT</samza.version>
112 |     </properties>
113 | 
114 |     <repositories>
115 |         <repository>
116 |             <id>my-local-repo</id>
117 |             <url>file://${user.home}/.m2/repository</url>
118 |         </repository>
119 |         <repository>
120 |             <id>apache-releases</id>
121 |             <url>https://repository.apache.org/content/groups/public</url>
122 |         </repository>
123 |         <repository>
124 |             <id>scala-tools.org</id>
125 |             <name>Scala-tools Maven2 Repository</name>
126 |             <url>https://oss.sonatype.org/content/groups/scala-tools</url>
127 |         </repository>
128 |     </repositories>
129 | 
130 |     <pluginRepositories>
131 |         <pluginRepository>
132 |             <id>scala-tools.org</id>
133 |             <name>Scala-tools Maven2 Repository</name>
134 |             <url>http://scala-tools.org/repo-releases</url>
135 |         </pluginRepository>
136 |     </pluginRepositories>
137 | 
138 |     <build>
139 | 
140 |         <plugins>
141 |             <plugin>
142 |                 <groupId>org.apache.maven.plugins</groupId>
143 |                 <artifactId>maven-compiler-plugin</artifactId>
144 |                 <version>3.1</version>
145 |                 <configuration>
146 |                     <source>1.7</source>
147 |                     <target>1.7</target>
148 |                 </configuration>
149 |             </plugin>
150 | 
151 |             <!-- plugin to build the tar.gz file filled with examples -->
152 |             <plugin>
153 |                 <artifactId>maven-assembly-plugin</artifactId>
154 |                 <version>2.3</version>
155 |                 <configuration>
156 |                     <descriptors>
157 |                         <descriptor>src/main/assembly/src.xml</descriptor>
158 |                     </descriptors>
159 |                 </configuration>
160 |                 <executions>
161 |                     <execution>
162 |                         <id>make-assembly</id>
163 |                         <phase>package</phase>
164 |                         <goals>
165 |                             <goal>single</goal>
166 |                         </goals>
167 |                     </execution>
168 |                 </executions>
169 |             </plugin>
170 | 
171 |             <plugin>
172 |                 <groupId>org.apache.maven.plugins</groupId>
173 |                 <artifactId>maven-shade-plugin</artifactId>
174 |                 <version>1.6</version>
175 |                 <configuration>
176 |                     <createDependencyReducedPom>true</createDependencyReducedPom>
177 |                     <filters>
178 |                         <filter>
179 |                             <artifact>*:*</artifact>
180 |                             <excludes>
181 |                                 <exclude>META-INF/*.SF</exclude>
182 |                                 <exclude>META-INF/*.DSA</exclude>
183 |                                 <exclude>META-INF/*.RSA</exclude>
184 |                             </excludes>
185 |                         </filter>
186 |                     </filters>
187 |                     <artifactSet>
188 |                         <excludes>
189 |                             <exclude>org.apache.samza:samza-shell</exclude>
190 |                         </excludes>
191 |                     </artifactSet>
192 |                 </configuration>
193 |                 <executions>
194 |                     <execution>
195 |                         <phase>package</phase>
196 |                         <goals>
197 |                             <goal>shade</goal>
198 |                         </goals>
199 |                         <configuration>
200 |                             <transformers>
201 |                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
202 |                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
203 |                                     <mainClass>uk.co.flax.samzaluwak.Interact</mainClass>
204 |                                 </transformer>
205 |                             </transformers>
206 |                         </configuration>
207 |                     </execution>
208 |                 </executions>
209 |             </plugin>
210 |         </plugins>
211 |     </build>
212 | 
213 | </project>


--------------------------------------------------------------------------------