├── zoie-core ├── .gitignore ├── src │ ├── main │ │ ├── resources │ │ │ └── .gitignore │ │ └── java │ │ │ └── proj │ │ │ └── zoie │ │ │ ├── hourglass │ │ │ ├── impl │ │ │ │ ├── HourGlassConfig.java │ │ │ │ ├── HourglassListener.java │ │ │ │ ├── CompositeHourglassListener.java │ │ │ │ └── Box.java │ │ │ ├── mbean │ │ │ │ └── HourglassAdminMBean.java │ │ │ └── api │ │ │ │ ├── HourglassIndexable.java │ │ │ │ └── HourglassIndexableInterpreter.java │ │ │ ├── store │ │ │ ├── ZoieStoreSerializer.java │ │ │ ├── AsyncZoieStoreConsumer.java │ │ │ ├── ZoieStore.java │ │ │ └── ZoieStoreConsumer.java │ │ │ ├── api │ │ │ ├── LifeCycleCotrolledDataConsumer.java │ │ │ ├── indexing │ │ │ │ ├── ZoieIndexableInterpreter.java │ │ │ │ ├── DefaultIndexingEventListener.java │ │ │ │ ├── AbstractZoieIndexableInterpreter.java │ │ │ │ ├── OptimizeScheduler.java │ │ │ │ ├── AbstractZoieIndexable.java │ │ │ │ ├── IndexingEventListener.java │ │ │ │ ├── IndexReaderDecorator.java │ │ │ │ ├── ZoieIndexable.java │ │ │ │ └── DefaultOptimizeScheduler.java │ │ │ ├── DocIDMapperFactory.java │ │ │ ├── ZoieHealth.java │ │ │ ├── Zoie.java │ │ │ ├── DataDoc.java │ │ │ ├── IndexCopier.java │ │ │ ├── DataProvider.java │ │ │ ├── DocIDMapper.java │ │ │ ├── impl │ │ │ │ ├── ZoieReaderContext.java │ │ │ │ ├── DefaultDocIDMapperFactory.java │ │ │ │ ├── util │ │ │ │ │ ├── IndexUtil.java │ │ │ │ │ ├── FileUtil.java │ │ │ │ │ ├── ArrayDocIdSet.java │ │ │ │ │ ├── ChannelUtil.java │ │ │ │ │ └── MemoryManager.java │ │ │ │ └── ZoieContext.java │ │ │ ├── ZoieException.java │ │ │ ├── ZoieExecutors.java │ │ │ ├── IndexReaderFactory.java │ │ │ ├── UIDFilter.java │ │ │ ├── DirectoryManager.java │ │ │ ├── UIDDocIdSet.java │ │ │ ├── ZoieThreadPoolExecutor.java │ │ │ └── DataConsumer.java │ │ │ ├── dataprovider │ │ │ └── jdbc │ │ │ │ ├── JDBCConnectionFactory.java │ │ │ │ ├── PreparedStatementBuilder.java │ │ │ │ ├── MysqlJDBCConnectionFactory.java │ │ │ │ ├── OracleJDBCConnectionFactory.java │ │ │ │ └── JDBCStreamDataProvider.java │ │ │ ├── impl │ │ │ └── indexing │ │ │ │ ├── ReaderCacheFactory.java │ │ │ │ ├── internal │ │ │ │ ├── RAMIndexFactory.java │ │ │ │ ├── DefaultRAMIndexFactory.java │ │ │ │ ├── DefaultRAMDiskIndexFactory.java │ │ │ │ ├── RAMLuceneIndexDataLoader.java │ │ │ │ ├── IndexSignature.java │ │ │ │ ├── ZoieIndexDeletionPolicy.java │ │ │ │ ├── DelegateIndexDataConsumer.java │ │ │ │ └── DiskIndexSnapshot.java │ │ │ │ ├── AbstractReaderCache.java │ │ │ │ ├── AbstractIndexReaderDecorator.java │ │ │ │ ├── DefaultIndexReaderDecorator.java │ │ │ │ ├── IndexingThread.java │ │ │ │ ├── SimpleReaderCache.java │ │ │ │ ├── NoopReaderCache.java │ │ │ │ ├── IndexUpdatedEvent.java │ │ │ │ ├── FileDataProvider.java │ │ │ │ └── MemoryStreamDataProvider.java │ │ │ ├── service │ │ │ └── api │ │ │ │ ├── ZoieSearchService.java │ │ │ │ ├── SearchRequest.java │ │ │ │ ├── SearchHit.java │ │ │ │ └── SearchResult.java │ │ │ ├── mbean │ │ │ ├── ZoieOptimizeSchedulerAdminMBean.java │ │ │ ├── ZoieIndexingStatusAdminMBean.java │ │ │ ├── ZoieSystemAdminMBean.java │ │ │ ├── DataProviderAdminMBean.java │ │ │ ├── ZoieOptimizeSchedulerAdmin.java │ │ │ ├── ZoieAdminMBean.java │ │ │ ├── DataProviderAdmin.java │ │ │ └── ZoieIndexingStatusAdmin.java │ │ │ └── cmdline │ │ │ └── JMXClient.java │ └── test │ │ └── java │ │ └── proj │ │ └── zoie │ │ └── test │ │ ├── data │ │ ├── DataForTests.java │ │ ├── InRangeDataInterpreterForTests.java │ │ └── DataInterpreterForTests.java │ │ ├── ZoieStoreTestCase.java │ │ ├── mock │ │ └── MockDataLoader.java │ │ └── PerfTest.java ├── resource │ └── log4j.properties └── pom.xml ├── zoie-jms ├── .gitignore ├── src │ └── main │ │ └── java │ │ └── proj │ │ └── zoie │ │ └── dataprovider │ │ └── jms │ │ ├── TopicFactory.java │ │ └── DataEventBuilder.java └── pom.xml ├── zoie-perf ├── .gitignore ├── bin │ ├── cleanup.sh │ └── run-perf.sh ├── src │ └── main │ │ └── java │ │ └── proj │ │ └── zoie │ │ └── perf │ │ ├── client │ │ ├── QueryHandler.java │ │ ├── ZoiePerfVersion.java │ │ ├── StoreQueryHandler.java │ │ ├── SearchQueryHandler.java │ │ └── TermFileBuilder.java │ │ ├── indexing │ │ ├── TweetInterpreter.java │ │ └── LinedFileDataProvider.java │ │ ├── servlet │ │ └── ZoiePerfServlet.java │ │ └── tools │ │ └── IndexReplicator.java ├── plot-script │ ├── indexing-latency.conf │ ├── index-event-rate.conf │ ├── index-size-rate.conf │ └── search.conf ├── conf │ └── perf.properties └── pom.xml ├── zoie-server ├── .gitignore └── pom.xml ├── zoie-example └── src │ └── main │ └── webapp │ ├── WEB-INF │ ├── conf │ │ ├── index.properties │ │ ├── applicationContext.spring │ │ ├── propertyLoader.spring │ │ ├── dataprovider.spring │ │ ├── jmx.spring │ │ └── zoie.spring │ ├── dwr.xml │ └── web.xml │ ├── styles │ └── style.css │ ├── index.html │ └── scripts │ └── search.js ├── contrib └── luke │ ├── lib │ └── lukeall-0.9.9.jar │ ├── xml │ └── zoie-luke.xml │ ├── build.xml │ └── java │ └── proj │ └── zoie │ └── tools │ └── luke │ └── ZoiePlugin.java ├── .gitignore ├── .settings ├── org.eclipse.jdt.ui.prefs ├── org.maven.ide.eclipse.prefs └── org.eclipse.jdt.core.prefs ├── README.md └── pom.xml /zoie-core/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /zoie-core/src/main/resources/.gitignore: -------------------------------------------------------------------------------- 1 | /git.properties 2 | -------------------------------------------------------------------------------- /zoie-jms/.gitignore: -------------------------------------------------------------------------------- 1 | /.classpath 2 | /target 3 | /.project 4 | -------------------------------------------------------------------------------- /zoie-perf/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.project 3 | /.classpath 4 | -------------------------------------------------------------------------------- /zoie-server/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.classpath 3 | /.project 4 | -------------------------------------------------------------------------------- /zoie-example/src/main/webapp/WEB-INF/conf/index.properties: -------------------------------------------------------------------------------- 1 | source.dir=src 2 | index.dir = index 3 | -------------------------------------------------------------------------------- /contrib/luke/lib/lukeall-0.9.9.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/senseidb/zoie/HEAD/contrib/luke/lib/lukeall-0.9.9.jar -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/hourglass/impl/HourGlassConfig.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.hourglass.impl; 2 | 3 | public class HourGlassConfig { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /zoie-perf/bin/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bin=`dirname "$0"` 4 | bin=`cd "$bin"; pwd` 5 | 6 | cd $bin/.. 7 | 8 | rm -rf idxDir 9 | rm -rf csvout 10 | -------------------------------------------------------------------------------- /zoie-perf/bin/run-perf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bin=`dirname "$0"` 4 | bin=`cd "$bin"; pwd` 5 | 6 | cd $bin/.. 7 | mvn -e exec:java -Dexec.mainClass=proj.zoie.perf.client.ZoiePerf 8 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/store/ZoieStoreSerializer.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.store; 2 | 3 | public interface ZoieStoreSerializer { 4 | long getUid(D data); 5 | 6 | byte[] toBytes(D data); 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build-test 2 | build 3 | dist 4 | ivy 5 | index 6 | logs 7 | output 8 | perf/perf-build 9 | lib 10 | MANIFEST.MF 11 | !lib/tools 12 | .settings 13 | *~ 14 | TAGS 15 | cscope.* 16 | /bin 17 | /.project 18 | -------------------------------------------------------------------------------- /zoie-perf/src/main/java/proj/zoie/perf/client/QueryHandler.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.perf.client; 2 | 3 | public interface QueryHandler { 4 | T handleQuery() throws Exception; 5 | 6 | String getCurrentVersion(); 7 | } 8 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/LifeCycleCotrolledDataConsumer.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api; 2 | 3 | public interface LifeCycleCotrolledDataConsumer extends DataConsumer { 4 | void start(); 5 | 6 | void stop(); 7 | } 8 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/hourglass/mbean/HourglassAdminMBean.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.hourglass.mbean; 2 | 3 | import proj.zoie.mbean.ZoieAdminMBean; 4 | 5 | public interface HourglassAdminMBean extends ZoieAdminMBean { 6 | } 7 | -------------------------------------------------------------------------------- /zoie-jms/src/main/java/proj/zoie/dataprovider/jms/TopicFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.dataprovider.jms; 2 | 3 | import javax.jms.Topic; 4 | 5 | public interface TopicFactory { 6 | 7 | public Topic createTopic(String name); 8 | } 9 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.ui.exception.name=e 3 | org.eclipse.jdt.ui.gettersetter.use.is=true 4 | org.eclipse.jdt.ui.keywordthis=false 5 | org.eclipse.jdt.ui.overrideannotation=true 6 | -------------------------------------------------------------------------------- /zoie-perf/plot-script/indexing-latency.conf: -------------------------------------------------------------------------------- 1 | set autoscale 2 | set title "indexing latency" 3 | set xlabel "time(sec)" 4 | set ylabel "index latency (ms)" 5 | set datafile separator "," 6 | plot 'csvout/indexLatency.csv' index 0 using 1:2 title 'indexing latency' with lines -------------------------------------------------------------------------------- /zoie-perf/plot-script/index-event-rate.conf: -------------------------------------------------------------------------------- 1 | set autoscale 2 | set title "indexing event rate" 3 | set xlabel "time(sec)" 4 | set ylabel "events per sec" 5 | set datafile separator "," 6 | plot 'csvout/consumeRateCount.csv' index 0 using 1:2 title 'events/sec' with linespoints -------------------------------------------------------------------------------- /zoie-perf/plot-script/index-size-rate.conf: -------------------------------------------------------------------------------- 1 | set autoscale 2 | set title "indexing event rate" 3 | set xlabel "time(sec)" 4 | set ylabel "MB per sec" 5 | set datafile separator "," 6 | plot 'csvout/consumeRateMB.csv' index 0 using 1:($2/1000000) title 'MB/sec' with linespoints 7 | -------------------------------------------------------------------------------- /zoie-core/resource/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console1 2 | 3 | log4j.appender.console1=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console1.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console1.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c] [%x] %m%n 6 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/indexing/ZoieIndexableInterpreter.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api.indexing; 2 | 3 | /** 4 | * Interface to translate from a data object to an indexable object. 5 | */ 6 | public interface ZoieIndexableInterpreter { 7 | ZoieIndexable convertAndInterpret(V src); 8 | } 9 | -------------------------------------------------------------------------------- /.settings/org.maven.ide.eclipse.prefs: -------------------------------------------------------------------------------- 1 | #Wed Mar 02 15:11:04 PST 2011 2 | activeProfiles=pom.xml 3 | eclipse.preferences.version=1 4 | fullBuildGoals=process-test-resources 5 | includeModules=false 6 | resolveWorkspaceProjects=true 7 | resourceFilterGoals=process-resources resources\:testResources 8 | skipCompilerPlugin=true 9 | version=1 10 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/DocIDMapperFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api; 2 | 3 | import java.io.IOException; 4 | 5 | public interface DocIDMapperFactory { 6 | DocIDMapper getDocIDMapper(ZoieSegmentReader reader) throws IOException; 7 | 8 | DocIDMapper getDocIDMapper(ZoieMultiReader reader) throws IOException; 9 | } 10 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/dataprovider/jdbc/JDBCConnectionFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.dataprovider.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | 6 | public interface JDBCConnectionFactory { 7 | Connection getConnection() throws SQLException; 8 | 9 | void showndown() throws SQLException; 10 | } 11 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/indexing/DefaultIndexingEventListener.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api.indexing; 2 | 3 | public class DefaultIndexingEventListener implements IndexingEventListener { 4 | 5 | public void handleIndexingEvent(IndexingEvent evt) { 6 | 7 | } 8 | 9 | public void handleUpdatedDiskVersion(String version) { 10 | 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /zoie-jms/src/main/java/proj/zoie/dataprovider/jms/DataEventBuilder.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.dataprovider.jms; 2 | 3 | import javax.jms.JMSException; 4 | import javax.jms.Message; 5 | 6 | import proj.zoie.api.DataConsumer.DataEvent; 7 | 8 | public interface DataEventBuilder { 9 | 10 | public DataEvent buildDataEvent(Message message) throws JMSException; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/impl/indexing/ReaderCacheFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.impl.indexing; 2 | 3 | import org.apache.lucene.index.IndexReader; 4 | 5 | import proj.zoie.api.IndexReaderFactory; 6 | 7 | public interface ReaderCacheFactory { 8 | public AbstractReaderCache newInstance( 9 | IndexReaderFactory readerfactory); 10 | } 11 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/store/AsyncZoieStoreConsumer.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.store; 2 | 3 | import java.util.Comparator; 4 | 5 | import proj.zoie.impl.indexing.AsyncDataConsumer; 6 | 7 | public class AsyncZoieStoreConsumer extends AsyncDataConsumer { 8 | 9 | public AsyncZoieStoreConsumer(Comparator versionComparator) { 10 | super(versionComparator); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/indexing/AbstractZoieIndexableInterpreter.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api.indexing; 2 | 3 | /** 4 | * @author "Xiaoyang Gu" 5 | * 6 | * @param The type of the data to be interpreted. 7 | */ 8 | public abstract class AbstractZoieIndexableInterpreter implements ZoieIndexableInterpreter { 9 | public abstract ZoieIndexable convertAndInterpret(V src); 10 | } 11 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/hourglass/api/HourglassIndexable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package proj.zoie.hourglass.api; 5 | 6 | import proj.zoie.api.indexing.AbstractZoieIndexable; 7 | 8 | /** 9 | * @author "Xiaoyang Gu" 10 | * 11 | */ 12 | public abstract class HourglassIndexable extends AbstractZoieIndexable { 13 | @Override 14 | public boolean isDeleted() { 15 | return false; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /zoie-example/src/main/webapp/WEB-INF/dwr.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/hourglass/impl/HourglassListener.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.hourglass.impl; 2 | 3 | import org.apache.lucene.index.IndexReader; 4 | 5 | import proj.zoie.api.Zoie; 6 | import proj.zoie.api.ZoieMultiReader; 7 | 8 | public interface HourglassListener { 9 | public void onNewZoie(Zoie zoie); 10 | 11 | public void onRetiredZoie(Zoie zoie); 12 | 13 | public void onIndexReaderCleanUp(ZoieMultiReader indexReader); 14 | } 15 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/ZoieHealth.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api; 2 | 3 | public class ZoieHealth { 4 | public static final long HEALTH_OK = 0; 5 | public static final long HEALTH_FATAL = 100; 6 | public static volatile long health = HEALTH_OK; 7 | 8 | public static void setFatal() { 9 | health = HEALTH_FATAL; 10 | } 11 | 12 | public static void setOK() { 13 | health = HEALTH_OK; 14 | } 15 | 16 | public static long getHealth() { 17 | return health; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/impl/indexing/internal/RAMIndexFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.impl.indexing.internal; 2 | 3 | import org.apache.lucene.index.IndexReader; 4 | 5 | import proj.zoie.api.indexing.IndexReaderDecorator; 6 | 7 | /** 8 | * @author "Xiaoyang Gu" 9 | * 10 | */ 11 | public abstract class RAMIndexFactory { 12 | public abstract RAMSearchIndex newInstance(String version, IndexReaderDecorator decorator, 13 | SearchIndexManager idxMgr); 14 | } 15 | -------------------------------------------------------------------------------- /zoie-example/src/main/webapp/WEB-INF/conf/applicationContext.spring: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/store/ZoieStore.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.store; 2 | 3 | import java.io.IOException; 4 | 5 | public interface ZoieStore { 6 | void put(long uid, byte[] data, String version) throws IOException; 7 | 8 | byte[] get(long uid) throws IOException; 9 | 10 | byte[][] get(long[] uids); 11 | 12 | String getVersion(); 13 | 14 | void delete(long uid, String version) throws IOException; 15 | 16 | void open() throws IOException; 17 | 18 | void close() throws IOException; 19 | 20 | void commit() throws IOException; 21 | } 22 | -------------------------------------------------------------------------------- /zoie-core/src/test/java/proj/zoie/test/data/DataForTests.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.test.data; 2 | 3 | public class DataForTests { 4 | public static String[] testdata = new String[] { "zoie odd 0", "zoie even 1", "zoie odd 2", 5 | "zoie even 3", "zoie odd 4", "zoie even 5", "zoie odd 6", "zoie even 7", "zoie odd 8", 6 | "zoie even 9" }; 7 | 8 | public static String[] testdata2 = new String[] { "zoie2 odd 0", "zoie2 even 1", "zoie2 odd 2", 9 | "zoie2 even 3", "zoie2 odd 4", "zoie2 even 5", "zoie2 odd 6", "zoie2 even 7", "zoie2 odd 8", 10 | "zoie2 even 9" }; 11 | } 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | What is Zoie 2 | =============== 3 | 4 | Zoie is a realtime search/indexing system written in Java. 5 | 6 | 7 | ------------------------------------ 8 | 9 | ### Wiki 10 | 11 | Wiki is available at: 12 | 13 | [http://linkedin.jira.com/wiki/display/ZOIE/Home](http://linkedin.jira.com/wiki/display/ZOIE/Home) 14 | 15 | ### Issues 16 | 17 | Issues are tracked at: 18 | 19 | [http://linkedin.jira.com/browse/ZOIE](http://linkedin.jira.com/browse/ZOIE) 20 | 21 | ### Release 22 | 23 | Maven: 24 | 25 | groupId: com.senseidb.zoie 26 | 27 | artifactId: zoie-core 28 | 29 | Latest Version: 3.0.0 30 | -------------------------------------------------------------------------------- /zoie-perf/plot-script/search.conf: -------------------------------------------------------------------------------- 1 | # min,max,mean,median,stddev,90%,95%,99% 2 | set autoscale 3 | set title "search performance" 4 | set xlabel "time(sec)" 5 | set ylabel "latency(ms)" 6 | set datafile separator "," 7 | plot "csvout/searchTimer.csv" index 0 using 1:4 title 'mean' with lines , \ 8 | "csvout/searchTimer.csv" index 0 using 1:5 title 'median' with lines , \ 9 | "csvout/searchTimer.csv" index 0 using 1:6 title 'stddev' with lines , \ 10 | "csvout/searchTimer.csv" index 0 using 1:7 title '90%' with lines , \ 11 | "csvout/searchTimer.csv" index 0 using 1:8 title '95%' with lines , \ 12 | "csvout/searchTimer.csv" index 0 using 1:9 title '99%' with lines -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | #Thu Jan 07 19:18:12 PST 2010 2 | eclipse.preferences.version=1 3 | org.eclipse.jdt.core.codeComplete.argumentPrefixes= 4 | org.eclipse.jdt.core.codeComplete.argumentSuffixes= 5 | org.eclipse.jdt.core.codeComplete.fieldPrefixes=_ 6 | org.eclipse.jdt.core.codeComplete.fieldSuffixes= 7 | org.eclipse.jdt.core.codeComplete.localPrefixes= 8 | org.eclipse.jdt.core.codeComplete.localSuffixes= 9 | org.eclipse.jdt.core.codeComplete.staticFieldPrefixes= 10 | org.eclipse.jdt.core.codeComplete.staticFieldSuffixes= 11 | org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= 12 | org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= 13 | -------------------------------------------------------------------------------- /zoie-example/src/main/webapp/WEB-INF/conf/propertyLoader.spring: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | /WEB-INF/conf/index.properties 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/Zoie.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api; 2 | 3 | import javax.management.StandardMBean; 4 | 5 | import org.apache.lucene.index.IndexReader; 6 | 7 | import proj.zoie.mbean.ZoieAdminMBean; 8 | 9 | public interface Zoie extends DataConsumer, 10 | IndexReaderFactory { 11 | void start(); 12 | 13 | void shutdown(); 14 | 15 | StandardMBean getStandardMBean(String name); 16 | 17 | String[] getStandardMBeanNames(); 18 | 19 | ZoieAdminMBean getAdminMBean(); 20 | 21 | void syncWithVersion(long timeInMillis, String version) throws ZoieException; 22 | 23 | void flushEvents(long timeout) throws ZoieException; 24 | } 25 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/impl/indexing/internal/DefaultRAMIndexFactory.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.impl.indexing.internal; 2 | 3 | import org.apache.lucene.index.IndexReader; 4 | import org.apache.lucene.store.RAMDirectory; 5 | 6 | import proj.zoie.api.indexing.IndexReaderDecorator; 7 | 8 | /** 9 | * @author "Xiaoyang Gu" 10 | * 11 | * @param 12 | */ 13 | public class DefaultRAMIndexFactory extends RAMIndexFactory { 14 | @Override 15 | public synchronized RAMSearchIndex newInstance(String version, 16 | IndexReaderDecorator decorator, SearchIndexManager idxMgr) { 17 | return new RAMSearchIndex(version, decorator, idxMgr, new RAMDirectory(), null); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/dataprovider/jdbc/PreparedStatementBuilder.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.dataprovider.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.sql.PreparedStatement; 5 | import java.sql.ResultSet; 6 | import java.sql.SQLException; 7 | 8 | import proj.zoie.api.DataConsumer.DataEvent; 9 | 10 | public interface PreparedStatementBuilder { 11 | PreparedStatement buildStatment(Connection conn, String fromVersion) throws SQLException; 12 | 13 | /** 14 | * The builder should not ever change the cursor of the result set. It should only work on the current row. 15 | * @param rs 16 | * @return DataEvent 17 | * @throws SQLException 18 | */ 19 | DataEvent buildDataEvent(ResultSet rs) throws SQLException; 20 | } 21 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/impl/indexing/AbstractReaderCache.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.impl.indexing; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.lucene.index.IndexReader; 6 | 7 | import proj.zoie.api.ZoieException; 8 | import proj.zoie.api.ZoieMultiReader; 9 | 10 | public abstract class AbstractReaderCache { 11 | public abstract List> getIndexReaders(); 12 | 13 | public abstract void returnIndexReaders(List> readers); 14 | 15 | public abstract void refreshCache(long timeout) throws ZoieException; 16 | 17 | public abstract void start(); 18 | 19 | public abstract void shutdown(); 20 | 21 | public abstract void setFreshness(long freshness); 22 | 23 | public abstract long getFreshness(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /zoie-example/src/main/webapp/WEB-INF/conf/dataprovider.spring: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /zoie-core/src/main/java/proj/zoie/api/DataDoc.java: -------------------------------------------------------------------------------- 1 | package proj.zoie.api; 2 | 3 | import org.apache.lucene.document.Document; 4 | 5 | import proj.zoie.api.indexing.AbstractZoieIndexable; 6 | 7 | public class DataDoc extends AbstractZoieIndexable { 8 | private long uid; 9 | private Document doc; 10 | private boolean valid; 11 | 12 | public DataDoc(long uid, Document doc) { 13 | this.uid = uid; 14 | this.doc = doc; 15 | this.valid = true; 16 | } 17 | 18 | public DataDoc(long uid) { 19 | this.uid = uid; 20 | this.valid = false; 21 | } 22 | 23 | @Override 24 | public IndexingReq[] buildIndexingReqs() { 25 | return new IndexingReq[] { new IndexingReq(doc) }; 26 | } 27 | 28 | @Override 29 | public long getUID() { 30 | return uid; 31 | } 32 | 33 | @Override 34 | public boolean isDeleted() { 35 | return !valid; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /zoie-perf/conf/perf.properties: -------------------------------------------------------------------------------- 1 | #test type, either zoie/nrt/store/feed 2 | perf.type=zoie 3 | 4 | perf.feed.batchsize=10000 5 | # index location 6 | perf.idxDir = idxDir 7 | 8 | # input file location 9 | perf.input=/home/jwang/dataset/tweets.txt 10 | 11 | # max number of input events to process 12 | perf.maxSize = 7500000000 13 | 14 | perf.maxEventsPerMin = 1000000 15 | 16 | perf.query.file=terms.txt 17 | 18 | perf.query.threads = 10 19 | 20 | # num millis to wait between search thread iterations 21 | perf.query.threadWait = 200 22 | 23 | #nrt test throttle factor, e.g. how often to load refresh index reader, 1000 = refresh once every second 24 | perf.nrt.throttle=1000 25 | 26 | # valid values are default,tier,zoie 27 | perf.nrt.mergePolicy=tier 28 | 29 | # trigger to turn on/off append-only mode 30 | perf.nrt.appendOnly=false 31 | 32 | perf.test.search = true 33 | 34 | perf.update.intervalSec = 2 35 | 36 | # valid values are file,mmap,nio 37 | 38 | perf.directory.type=mmap 39 | -------------------------------------------------------------------------------- /contrib/luke/xml/zoie-luke.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |