├── .gitignore ├── .gitmodules ├── CHANGES.txt ├── LICENSE.txt ├── README.markdown ├── build.xml ├── doc └── custom_server.txt ├── lib └── build │ ├── ant-contrib-1.0b3.jar │ ├── ant-eclipse-1.0-jvm1.2.jar │ ├── antlr-3.2.jar │ ├── asm-3.0.jar │ ├── asm-tree-3.0.jar │ ├── checkstyle-5.2.jar │ ├── cobertura-1.9.3.jar │ ├── commons-beanutils-core-1.8.3.jar │ ├── commons-logging-1.0.4.jar │ ├── commons-logging-api-1.0.4.jar │ ├── guava-r07.jar │ ├── log4j-1.2.15.jar │ └── oro-2.0.8.jar ├── modules ├── katta-core │ ├── bin │ │ ├── katta │ │ ├── katta-config.sh │ │ ├── katta-daemon.sh │ │ ├── katta-daemons.sh │ │ ├── nodes.sh │ │ ├── start-all.sh │ │ └── stop-all.sh │ ├── build.properties │ ├── build.xml │ ├── conf │ │ ├── katta-env.sh │ │ ├── katta.ec2.properties.template │ │ ├── katta.master.properties │ │ ├── katta.node.properties │ │ ├── katta.zk.properties │ │ ├── log4j.properties │ │ └── nodes │ ├── lib │ │ ├── compile │ │ │ ├── commons-cli-1.2.jar │ │ │ ├── commons-math-1.2.jar │ │ │ ├── guava-11.0.1-sources.jar │ │ │ ├── guava-11.0.1.jar │ │ │ ├── hadoop │ │ │ │ ├── commons-codec-1.3.jar │ │ │ │ ├── commons-httpclient-3.0.1.jar │ │ │ │ ├── commons-logging-1.0.4.jar │ │ │ │ ├── commons-logging-api-1.0.4.jar │ │ │ │ ├── commons-net-1.4.1.jar │ │ │ │ ├── hadoop-0.20.2-core.jar │ │ │ │ ├── jets3t-0.6.1.jar │ │ │ │ └── xmlenc-0.52.jar │ │ │ ├── jetty-6.1.14.jar │ │ │ ├── jetty-util-6.1.14.jar │ │ │ ├── jsch-0.1.41.jar │ │ │ ├── jsp-2.1.jar │ │ │ ├── jsp-api-2.1.jar │ │ │ ├── log4j-1.2.15.jar │ │ │ ├── lucene-core-3.5.0-sources.jar │ │ │ ├── lucene-core-3.5.0.jar │ │ │ ├── servlet-api-2.4.jar │ │ │ ├── typica-1.5.1.jar │ │ │ ├── zkclient-0.2-sources.jar │ │ │ ├── zkclient-0.2.jar │ │ │ └── zookeeper-3.3.2.jar │ │ └── test │ │ │ ├── fest-assert-1.3-sources.jar │ │ │ ├── fest-assert-1.3.jar │ │ │ ├── fest-util-1.1.4-sources.jar │ │ │ ├── fest-util-1.1.4.jar │ │ │ ├── hadoop-0.20.2-test.jar │ │ │ ├── hamcrest-core-1.1.jar │ │ │ ├── hamcrest-library-1.1.jar │ │ │ ├── junit-4.10-sources.jar │ │ │ ├── junit-4.10.jar │ │ │ ├── mockito-core-1.8.1.jar │ │ │ └── objenesis-1.0.jar │ └── src │ │ ├── it │ │ └── java │ │ │ └── net │ │ │ └── sf │ │ │ └── katta │ │ │ └── integrationTest │ │ │ ├── ClientIntegrationTest.java │ │ │ ├── DeployPolicyIntegrationTest.java │ │ │ ├── FailoverTest.java │ │ │ ├── MasterIntegrationTest.java │ │ │ ├── MultiClusterTest.java │ │ │ ├── NodeIntegrationTest.java │ │ │ ├── NodeReconnectWhileStartingTest.java │ │ │ ├── ShardAccessAfterUndeployTest.java │ │ │ ├── lib │ │ │ ├── lucene │ │ │ │ ├── Hit_MatchLuceneTest.java │ │ │ │ ├── LuceneClientFailoverTest.java │ │ │ │ ├── LuceneClientTest.java │ │ │ │ ├── LuceneComplianceTest.java │ │ │ │ ├── LuceneSearchIntegrationTest.java │ │ │ │ └── LuceneSearchPerformanceTest.java │ │ │ ├── mapfile │ │ │ │ └── MapFileClientTest.java │ │ │ └── sleep │ │ │ │ └── SleepClientTest.java │ │ │ ├── loadtest │ │ │ └── LuceneLoadIntegrationTest.java │ │ │ ├── manuell │ │ │ └── DeployUndeploySearchInLoop.java │ │ │ └── support │ │ │ ├── AbstractIntegrationTest.java │ │ │ ├── ClusterRule.java │ │ │ ├── HadoopMiniCluster.java │ │ │ └── KattaMiniCluster.java │ │ ├── main │ │ ├── java │ │ │ └── net │ │ │ │ └── sf │ │ │ │ └── katta │ │ │ │ ├── DefaultNameSpaceImpl.java │ │ │ │ ├── Katta.java │ │ │ │ ├── client │ │ │ │ ├── BasicNodeSelectionPolicy.java │ │ │ │ ├── Client.java │ │ │ │ ├── ClientResult.java │ │ │ │ ├── DeployClient.java │ │ │ │ ├── IDeployClient.java │ │ │ │ ├── IIndexDeployFuture.java │ │ │ │ ├── INodeExecutor.java │ │ │ │ ├── INodeProxyManager.java │ │ │ │ ├── INodeSelectionPolicy.java │ │ │ │ ├── IResultPolicy.java │ │ │ │ ├── IResultReceiver.java │ │ │ │ ├── IndexDeployFuture.java │ │ │ │ ├── IndexState.java │ │ │ │ ├── NodeInteraction.java │ │ │ │ ├── NodeProxyManager.java │ │ │ │ ├── ResultCompletePolicy.java │ │ │ │ ├── ShardAccessException.java │ │ │ │ ├── ShuffleNodeSelectionPolicy.java │ │ │ │ └── WorkQueue.java │ │ │ │ ├── index │ │ │ │ └── IndexMetaData.java │ │ │ │ ├── lib │ │ │ │ ├── lucene │ │ │ │ │ ├── DefaultSearcherFactory.java │ │ │ │ │ ├── DocumentFrequencyWritable.java │ │ │ │ │ ├── FieldSortComparator.java │ │ │ │ │ ├── FilterWritable.java │ │ │ │ │ ├── Hit.java │ │ │ │ │ ├── Hits.java │ │ │ │ │ ├── HitsMapWritable.java │ │ │ │ │ ├── ILuceneClient.java │ │ │ │ │ ├── ILuceneServer.java │ │ │ │ │ ├── ISeacherFactory.java │ │ │ │ │ ├── LuceneClient.java │ │ │ │ │ ├── LuceneServer.java │ │ │ │ │ ├── QueryWritable.java │ │ │ │ │ ├── SortWritable.java │ │ │ │ │ └── TermWritable.java │ │ │ │ └── mapfile │ │ │ │ │ ├── IMapFileClient.java │ │ │ │ │ ├── IMapFileServer.java │ │ │ │ │ ├── MapFileClient.java │ │ │ │ │ ├── MapFileServer.java │ │ │ │ │ └── TextArrayWritable.java │ │ │ │ ├── master │ │ │ │ ├── DefaultDistributionPolicy.java │ │ │ │ ├── HostAwareDistributionPolicy.java │ │ │ │ ├── IDeployPolicy.java │ │ │ │ ├── LowestShardCountDistributionPolicy.java │ │ │ │ ├── Master.java │ │ │ │ ├── MasterContext.java │ │ │ │ ├── OperationRegistry.java │ │ │ │ ├── OperationWatchdog.java │ │ │ │ └── OperatorThread.java │ │ │ │ ├── node │ │ │ │ ├── IContentServer.java │ │ │ │ ├── Node.java │ │ │ │ ├── NodeContext.java │ │ │ │ ├── ShardManager.java │ │ │ │ └── monitor │ │ │ │ │ ├── IMonitor.java │ │ │ │ │ ├── JmxMonitor.java │ │ │ │ │ ├── MetricLogger.java │ │ │ │ │ └── MetricsRecord.java │ │ │ │ ├── operation │ │ │ │ ├── OperationId.java │ │ │ │ ├── master │ │ │ │ │ ├── AbstractIndexOperation.java │ │ │ │ │ ├── BalanceIndexOperation.java │ │ │ │ │ ├── CheckIndicesOperation.java │ │ │ │ │ ├── IndexDeployException.java │ │ │ │ │ ├── IndexDeployOperation.java │ │ │ │ │ ├── IndexReinitializeOperation.java │ │ │ │ │ ├── IndexUndeployOperation.java │ │ │ │ │ ├── MasterOperation.java │ │ │ │ │ └── RemoveObsoleteShardsOperation.java │ │ │ │ └── node │ │ │ │ │ ├── AbstractShardOperation.java │ │ │ │ │ ├── DeployResult.java │ │ │ │ │ ├── NodeOperation.java │ │ │ │ │ ├── OperationResult.java │ │ │ │ │ ├── ShardDeployOperation.java │ │ │ │ │ ├── ShardRedeployOperation.java │ │ │ │ │ └── ShardUndeployOperation.java │ │ │ │ ├── protocol │ │ │ │ ├── BlockingQueue.java │ │ │ │ ├── ConnectedComponent.java │ │ │ │ ├── IAddRemoveListener.java │ │ │ │ ├── InteractionProtocol.java │ │ │ │ ├── MasterQueue.java │ │ │ │ ├── NodeQueue.java │ │ │ │ ├── ReplicationReport.java │ │ │ │ ├── metadata │ │ │ │ │ ├── IndexDeployError.java │ │ │ │ │ ├── IndexMetaData.java │ │ │ │ │ ├── MasterMetaData.java │ │ │ │ │ ├── NodeMetaData.java │ │ │ │ │ └── Version.java │ │ │ │ └── upgrade │ │ │ │ │ ├── UpgradeAction.java │ │ │ │ │ ├── UpgradeAction05_06.java │ │ │ │ │ └── UpgradeRegistry.java │ │ │ │ ├── tool │ │ │ │ ├── SampleIndexGenerator.java │ │ │ │ ├── ZkTool.java │ │ │ │ ├── ec2 │ │ │ │ │ ├── Ec2Configuration.java │ │ │ │ │ ├── Ec2Instance.java │ │ │ │ │ ├── Ec2Service.java │ │ │ │ │ └── SshUtil.java │ │ │ │ └── loadtest │ │ │ │ │ ├── LoadTestMasterOperation.java │ │ │ │ │ ├── LoadTestNodeOperation.java │ │ │ │ │ ├── LoadTestNodeOperationResult.java │ │ │ │ │ ├── LoadTestQueryResult.java │ │ │ │ │ └── query │ │ │ │ │ ├── AbstractQueryExecutor.java │ │ │ │ │ ├── LuceneSearchExecutor.java │ │ │ │ │ └── MapfileAccessExecutor.java │ │ │ │ └── util │ │ │ │ ├── CircularList.java │ │ │ │ ├── ClassUtil.java │ │ │ │ ├── ClientConfiguration.java │ │ │ │ ├── CollectionUtil.java │ │ │ │ ├── ConcurrentOne2ManyListMap.java │ │ │ │ ├── DefaultDateFormat.java │ │ │ │ ├── FileUtil.java │ │ │ │ ├── HadoopUtil.java │ │ │ │ ├── IHadoopConstants.java │ │ │ │ ├── KattaConfiguration.java │ │ │ │ ├── KattaException.java │ │ │ │ ├── MasterConfiguration.java │ │ │ │ ├── MergeSort.java │ │ │ │ ├── NodeConfiguration.java │ │ │ │ ├── One2ManyListMap.java │ │ │ │ ├── PropertyUtil.java │ │ │ │ ├── StringUtil.java │ │ │ │ ├── ThrottledInputStream.java │ │ │ │ ├── WebApp.java │ │ │ │ ├── WritableType.java │ │ │ │ ├── ZkConfiguration.java │ │ │ │ └── ZkKattaUtil.java │ │ └── resources │ │ │ └── xpath.properties │ │ └── test │ │ ├── java │ │ └── net │ │ │ └── sf │ │ │ └── katta │ │ │ ├── AbstractTest.java │ │ │ ├── AbstractWritableTest.java │ │ │ ├── AbstractZkTest.java │ │ │ ├── KattaTest.java │ │ │ ├── client │ │ │ ├── BasicNodeSelectionPolicyTest.java │ │ │ ├── ClientResultTest.java │ │ │ ├── ClientTest.java │ │ │ ├── DeployClientTest.java │ │ │ ├── DeployClientZkTest.java │ │ │ ├── IndexDeployFutureTest.java │ │ │ ├── NodeInteractionTest.java │ │ │ ├── NodeProxyManagerTest.java │ │ │ ├── ResultCompletePolicyTest.java │ │ │ └── WorkQueueTest.java │ │ │ ├── lib │ │ │ ├── lucene │ │ │ │ ├── DocumentFrequencyWritableTest.java │ │ │ │ ├── FieldSortComparatorTest.java │ │ │ │ ├── FilterWritableTest.java │ │ │ │ ├── HitSortPerformanceTest.java │ │ │ │ ├── HitTest.java │ │ │ │ ├── HitsMapWritableTest.java │ │ │ │ ├── LuceneServerTest.java │ │ │ │ ├── LuceneServer_KattaHitQueueTest.java │ │ │ │ └── QueryWritableTest.java │ │ │ └── mapfile │ │ │ │ └── MapFileServerTest.java │ │ │ ├── master │ │ │ ├── DefaultDistributionPolicyTest.java │ │ │ ├── HostAwareDistributionPolicyTest.java │ │ │ ├── LowestShardCountDistributionPolicyTest.java │ │ │ ├── MasterMockTest.java │ │ │ ├── MasterZkTest.java │ │ │ ├── OperationRegistryTest.java │ │ │ ├── OperationWatchdogTest.java │ │ │ └── OperatorThreadTest.java │ │ │ ├── node │ │ │ ├── NodeMockTest.java │ │ │ ├── NodeOperationProcessorTest.java │ │ │ ├── NodeZkTest.java │ │ │ ├── ShardManagerTest.java │ │ │ ├── SleepServerTest.java │ │ │ └── monitor │ │ │ │ ├── JmxMonitorTest.java │ │ │ │ └── MetricLoggerTest.java │ │ │ ├── operation │ │ │ ├── master │ │ │ │ ├── AbstractMasterNodeZkTest.java │ │ │ │ ├── BalanceIndexOperationTest.java │ │ │ │ ├── CheckIndicesOperationTest.java │ │ │ │ ├── IndexDeployOperationTest.java │ │ │ │ ├── IndexUndeployOperationTest.java │ │ │ │ └── RemoveObsoleteShardsOperationTest.java │ │ │ └── node │ │ │ │ ├── AbstractNodeOperationMockTest.java │ │ │ │ ├── ShardDeployOperationTest.java │ │ │ │ ├── ShardRedeployOperationTest.java │ │ │ │ └── ShardUndeployOperationTest.java │ │ │ ├── protocol │ │ │ ├── BlockingQueueTest.java │ │ │ ├── InteractionProtocolTest.java │ │ │ ├── MasterQueueTest.java │ │ │ ├── NodeQueueTest.java │ │ │ └── upgrade │ │ │ │ ├── UpgradeAction05_06Test.java │ │ │ │ └── UpgradeRegistryTest.java │ │ │ ├── testutil │ │ │ ├── GenerateMapFiles.java │ │ │ ├── Mocks.java │ │ │ ├── PrintMethodNames.java │ │ │ ├── TestIoUtil.java │ │ │ ├── TestResources.java │ │ │ ├── TestUtil.java │ │ │ ├── ZkTestSystem.java │ │ │ └── mockito │ │ │ │ ├── AlmostVerificationMode.java │ │ │ │ ├── ChainedAnswer.java │ │ │ │ ├── PauseAnswer.java │ │ │ │ ├── PrintStacktraceAnswer.java │ │ │ │ ├── SerializableCountDownLatchAnswer.java │ │ │ │ ├── SerializableThrowsExceptionAnswer.java │ │ │ │ ├── SleepingAnswer.java │ │ │ │ └── WaitingAnswer.java │ │ │ ├── tool │ │ │ ├── SampleIndexGeneratorTest.java │ │ │ ├── ec2 │ │ │ │ └── Ec2ServiceTest.java │ │ │ └── loadtest │ │ │ │ ├── LoadTestMasterOperationTest.java │ │ │ │ └── LoadTestNodeOperationTest.java │ │ │ └── util │ │ │ ├── CircularListTest.java │ │ │ ├── ClassUtilTest.java │ │ │ ├── CollectionUtilTest.java │ │ │ ├── ConcurrentOne2ManyListMapTest.java │ │ │ ├── FileUtilTest.java │ │ │ ├── ISleepClient.java │ │ │ ├── ISleepServer.java │ │ │ ├── One2ManyListMapTest.java │ │ │ ├── SleepClient.java │ │ │ ├── SleepServer.java │ │ │ ├── ThrottledInputStreamTest.java │ │ │ ├── WritableTypeTest.java │ │ │ └── ZkConfigurationTest.java │ │ ├── resources │ │ ├── katta.master.properties │ │ ├── katta.node.properties │ │ ├── katta.zk.properties │ │ ├── katta.zk.properties_alt_root │ │ ├── log4j.properties │ │ ├── user.xml │ │ └── xpath.properties │ │ ├── testIndexA │ │ ├── aIndex │ │ │ ├── _0.cfs │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ ├── bIndex │ │ │ ├── _0.cfs │ │ │ ├── copy.done │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ ├── cIndex │ │ │ ├── _0.cfs │ │ │ ├── copy.done │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ └── dIndex │ │ │ ├── _us.cfs │ │ │ └── segments │ │ ├── testIndexB │ │ ├── aIndex.zip │ │ ├── bIndex.zip │ │ ├── cIndex.zip │ │ └── dindex.zip │ │ ├── testIndexC │ │ └── aIndex │ │ │ ├── _0.cfs │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ ├── testIndexInvalid │ │ ├── aIndex │ │ │ └── _0.cfs │ │ ├── bIndex │ │ │ ├── _0.cfs │ │ │ ├── copy.done │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ ├── cIndex │ │ │ ├── _0.cfs │ │ │ ├── copy.done │ │ │ ├── segments.gen │ │ │ └── segments_3 │ │ └── dIndex │ │ │ ├── _us.cfs │ │ │ └── segments │ │ ├── testMapFileA │ │ ├── a1 │ │ │ ├── data │ │ │ └── index │ │ ├── a2 │ │ │ ├── data │ │ │ └── index │ │ ├── a3 │ │ │ ├── data │ │ │ └── index │ │ └── a4 │ │ │ ├── data │ │ │ └── index │ │ └── testMapFileB │ │ ├── b1 │ │ ├── data │ │ └── index │ │ └── b2 │ │ ├── data │ │ └── index ├── katta-ec2 │ ├── README.txt │ └── bin │ │ ├── cmd-katta-cluster │ │ ├── create-katta-image │ │ ├── create-nfs-share │ │ ├── delete-katta-cluster │ │ ├── image │ │ ├── create-katta-image-remote │ │ └── ec2-run-user-data │ │ ├── katta-ec2 │ │ ├── katta-ec2-env.sh.template │ │ ├── katta-ec2-init-remote.sh │ │ ├── launch-katta-cluster │ │ ├── launch-katta-master │ │ ├── launch-katta-nodes │ │ ├── list-katta-clusters │ │ └── terminate-katta-cluster └── katta-indexing-sample │ ├── build.properties │ ├── build.xml │ ├── run.sh │ └── src │ ├── main │ └── java │ │ └── net │ │ └── sf │ │ └── katta │ │ └── indexing │ │ ├── IndexerJob.java │ │ └── SequenceFileCreator.java │ └── test │ └── java │ └── net │ └── sf │ └── katta │ └── indexing │ ├── IndexerJobTest.java │ ├── SearchPathUtil.java │ └── SequnceFileCreatorTest.java ├── sample-data └── texts │ └── alice.txt ├── src └── build │ ├── ant │ ├── build-all.xml │ ├── build.properties │ └── common-build.xml │ ├── checkstyle │ ├── checkstyle-noframes.xsl │ ├── checkstyle.xml │ └── java.header │ └── eclipse │ ├── org.eclipse.jdt.core.prefs │ ├── org.eclipse.jdt.ui.prefs │ └── org.moreunit.prefs └── trash ├── IZkChildListener.java ├── IZkDataListener.java ├── IZkReconnectListener.java ├── NumberPaddingUtil.java ├── SymlinkResourceLoader.java ├── ZKClient.java ├── ZKClientTest.java ├── ZkClientReconnectTest.java ├── ZkEventType.java ├── ZkPathsTest.java ├── ZkServer.java └── ZkServerTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | zookeeper-data/ 2 | zookeeper-log-data/ 3 | .project 4 | .classpath 5 | .settings 6 | /build/ 7 | /modules/*/build/ 8 | prodDb* 9 | .DS_Store 10 | stacktrace.log 11 | log.log 12 | results 13 | extras/ec2/bin/katta-ec2-env.sh 14 | extras/katta.gui/katta.gui-0.1.war 15 | modules/katta-ec2/bin/katta-ec2-env.sh 16 | modules/katta-gui 17 | .idea 18 | *.iml 19 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "modules/zkclient"] 2 | path = modules/zkclient 3 | #url = http://github.com/joa23/zkclient.git 4 | url = git://github.com/joa23/zkclient.git 5 | #url = git@github.com:joa23/zkclient.git 6 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Katta: Lucene in the cloud! 2 | ===== 3 | **(Or Hadoop MapFiles... Or your own custom shardable content...!)** 4 | 5 | + Website: http://katta.sourceforge.net/ 6 | + Downloads: http://sourceforge.net/project/showfiles.php?group_id=225750 7 | + Documentation: http://katta.sourceforge.net/documentation 8 | + Mailing List: https://lists.sourceforge.net/lists/listinfo/katta-developer 9 | + Issue Tracking: http://oss.101tec.com/jira/browse/KATTA 10 | + Sources: https://github.com/sgroschupf/katta 11 | + Change-Log: http://oss.101tec.com/jira/browse/KATTA?report=com.atlassian.jira.plugin.system.project:changelog-panel 12 | + Apache 2.0 License 13 | 14 | 15 | Currently active branches: 16 | ===== 17 | + master (v0.7) 18 | + v0.6 (0.6.5) 19 | 20 | 21 | Build Katta from sources: 22 | ===== 23 | 24 | + git clone https://github.com/sgroschupf/katta.git 25 | + ant test 26 | 27 | 28 | Operate Katta 29 | ===== 30 | 31 | + To start/stop a katta cluster: 32 | + $ bin/start-all.sh 33 | + $ bin/stop-all.sh 34 | + To interact with a katta cluster: 35 | + $ bin/katta 36 | 37 | -------------------------------------------------------------------------------- /lib/build/ant-contrib-1.0b3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/ant-contrib-1.0b3.jar -------------------------------------------------------------------------------- /lib/build/ant-eclipse-1.0-jvm1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/ant-eclipse-1.0-jvm1.2.jar -------------------------------------------------------------------------------- /lib/build/antlr-3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/antlr-3.2.jar -------------------------------------------------------------------------------- /lib/build/asm-3.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/asm-3.0.jar -------------------------------------------------------------------------------- /lib/build/asm-tree-3.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/asm-tree-3.0.jar -------------------------------------------------------------------------------- /lib/build/checkstyle-5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/checkstyle-5.2.jar -------------------------------------------------------------------------------- /lib/build/cobertura-1.9.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/cobertura-1.9.3.jar -------------------------------------------------------------------------------- /lib/build/commons-beanutils-core-1.8.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/commons-beanutils-core-1.8.3.jar -------------------------------------------------------------------------------- /lib/build/commons-logging-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/commons-logging-1.0.4.jar -------------------------------------------------------------------------------- /lib/build/commons-logging-api-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/commons-logging-api-1.0.4.jar -------------------------------------------------------------------------------- /lib/build/guava-r07.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/guava-r07.jar -------------------------------------------------------------------------------- /lib/build/log4j-1.2.15.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/log4j-1.2.15.jar -------------------------------------------------------------------------------- /lib/build/oro-2.0.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/lib/build/oro-2.0.8.jar -------------------------------------------------------------------------------- /modules/katta-core/bin/katta-config.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # included in all the katta scripts with source command 17 | # should not be executable directly 18 | # also should not be passed any arguments, since we need original $* 19 | 20 | # resolve links - $0 may be a softlink 21 | 22 | unset CDPATH 23 | this="$0" 24 | while [ -h "$this" ]; do 25 | ls=`ls -ld "$this"` 26 | link=`expr "$ls" : '.*-> \(.*\)$'` 27 | if expr "$link" : '.*/.*' > /dev/null; then 28 | this="$link" 29 | else 30 | this=`dirname "$this"`/"$link" 31 | fi 32 | done 33 | 34 | # convert relative path to absolute path 35 | bin=`dirname "$this"` 36 | script=`basename "$this"` 37 | bin=`cd "$bin"; pwd` 38 | this="$bin/$script" 39 | 40 | # the root of the Katta installation 41 | export KATTA_HOME=`dirname "$this"`/.. 42 | 43 | # check to see if the conf dir is given as an optional argument 44 | if [ $# -gt 1 ] 45 | then 46 | if [ "--config" = "$1" ] 47 | then 48 | shift 49 | confdir=$1 50 | shift 51 | KATTA_CONF_DIR=$confdir 52 | fi 53 | fi 54 | 55 | # Allow alternate conf dir location. 56 | KATTA_CONF_DIR="${KATTA_CONF_DIR:-$KATTA_HOME/conf}" 57 | 58 | #check to see it is specified whether to use the nodes or the 59 | # masters file 60 | if [ $# -gt 1 ] 61 | then 62 | if [ "--hosts" = "$1" ] 63 | then 64 | shift 65 | nodesfile=$1 66 | shift 67 | export KATTA_NODES="${KATTA_CONF_DIR}/$nodesfile" 68 | fi 69 | fi 70 | -------------------------------------------------------------------------------- /modules/katta-core/bin/katta-daemons.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Run a Katta command on all node hosts. 20 | 21 | usage="Usage: katta-daemons.sh [--start-node ] [--num-nodes ] [--config confdir] [--hosts hostlistfile] [start|stop] command args..." 22 | 23 | unset CDPATH 24 | # if no args specified, show usage 25 | if [ $# -le 1 ]; then 26 | echo $usage 27 | exit 1 28 | fi 29 | 30 | bin=`dirname "$0"` 31 | bin=`cd "$bin"; pwd` 32 | 33 | . $bin/katta-config.sh 34 | 35 | exec "$bin/nodes.sh" --config $KATTA_CONF_DIR cd "$KATTA_HOME" \; "$bin/katta-daemon.sh" --config $KATTA_CONF_DIR "$@" 36 | -------------------------------------------------------------------------------- /modules/katta-core/bin/start-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Start all katta daemons. Run this on master node. 20 | 21 | unset CDPATH 22 | bin=`dirname "$0"` 23 | bin=`cd "$bin"; pwd` 24 | 25 | . "$bin"/katta-config.sh 26 | 27 | # start master daemons 28 | "$bin"/katta-daemon.sh start startMaster --config $KATTA_CONF_DIR 29 | 30 | sleep 10 31 | 32 | # start slave daemons 33 | "$bin"/katta-daemons.sh start startNode --config $KATTA_CONF_DIR 34 | -------------------------------------------------------------------------------- /modules/katta-core/bin/stop-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Stop all katta daemons. Run this on master node. 20 | 21 | unset CDPATH 22 | bin=`dirname "$0"` 23 | bin=`cd "$bin"; pwd` 24 | 25 | . "$bin"/katta-config.sh 26 | 27 | # stop master daemons 28 | "$bin"/katta-daemon.sh stop startMaster --config $KATTA_CONF_DIR 29 | 30 | # stop node daemons 31 | "$bin"/katta-daemons.sh stop startNode --config $KATTA_CONF_DIR 32 | -------------------------------------------------------------------------------- /modules/katta-core/build.properties: -------------------------------------------------------------------------------- 1 | root.dir=../.. 2 | -------------------------------------------------------------------------------- /modules/katta-core/conf/katta-env.sh: -------------------------------------------------------------------------------- 1 | # Set Katta-specific environment variables here. 2 | 3 | # The only required environment variable is JAVA_HOME. All others are 4 | # optional. When running a distributed configuration it is best to 5 | # set JAVA_HOME in this file, so that it is correctly defined on 6 | # remote nodes. 7 | 8 | # The java implementation to use. Required. 9 | # export JAVA_HOME=/usr/lib/j2sdk1.5-sun 10 | 11 | # Extra Java CLASSPATH elements. Optional. 12 | # export KATTA_CLASSPATH= 13 | 14 | # The maximum amount of heap to use, in MB. Default is 1000. 15 | # export KATTA_HEAPSIZE=2000 16 | 17 | # Extra Java runtime options. Empty by default. 18 | # export KATTA_OPTS=-server 19 | 20 | # Extra ssh options. Empty by default. 21 | # export KATTA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=KATTA_CONF_DIR" 22 | 23 | # Where log files are stored. $KATTA_HOME/logs by default. 24 | # export KATTA_LOG_DIR=${KATTA_HOME}/logs 25 | 26 | # File naming remote node hosts. $KATTA_HOME/conf/nodes by default. 27 | # export KATTA_NODES=${KATTA_HOME}/conf/nodes 28 | 29 | # host:path where katta code should be rsync'd from. Unset by default. 30 | # export KATTA_MASTER=master:/home/$USER/src/katta-version 31 | 32 | # Seconds to sleep between node commands. Unset by default. This 33 | # can be useful in large clusters, where, e.g., node rsyncs can 34 | # otherwise arrive faster than the master can service them. 35 | # export KATTA_NODE_SLEEP=0.1 36 | 37 | # The directory where pid files are stored. /tmp by default. 38 | # export KATTA_PID_DIR=/var/katta/pids 39 | 40 | # A string representing this instance of katta. $USER by default. 41 | # export KATTA_IDENT_STRING=$USER 42 | 43 | # The scheduling priority for daemon processes. See 'man nice'. 44 | # export KATTA_NICENESS=10 45 | 46 | # The level of logging. Possible values are Debug, Info, Error, Warn and None. 47 | export KATTA_LOG_LEVEL=Debug 48 | -------------------------------------------------------------------------------- /modules/katta-core/conf/katta.ec2.properties.template: -------------------------------------------------------------------------------- 1 | # configuration file to setup ec2 account 2 | 3 | # aws account id 4 | aws.accountId= 5 | # aws access id 6 | aws.accessKeyId= 7 | # aws secret access key 8 | aws.secretAccessKey= 9 | # name of the key pair that should be used to start ec2 instances. 10 | aws.keyPairName=katta-keypair 11 | #path to the scret key for the configured keyPairName 12 | aws.keyPath= 13 | # base image, an ec2 image id that at least has java installed. 14 | aws.aim=ami-45e7002c 15 | -------------------------------------------------------------------------------- /modules/katta-core/conf/katta.master.properties: -------------------------------------------------------------------------------- 1 | # used distribution policy 2 | master.deploy.policy=net.sf.katta.master.DefaultDistributionPolicy 3 | #maximum time the master stays in safemode on startup, waiting on expected nodes 4 | safemode.maxTime=20000 5 | -------------------------------------------------------------------------------- /modules/katta-core/conf/katta.node.properties: -------------------------------------------------------------------------------- 1 | # the start port to try 2 | node.server.port.start = 20000 3 | # local folder on node where shards will be stored during serving 4 | node.shard.folder=/tmp/katta-shards 5 | node.monitor.class=net.sf.katta.node.monitor.JmxMonitor 6 | node.server.class=net.sf.katta.lib.lucene.LuceneServer 7 | 8 | # uncomment to enable throttling of bandwith in kilo-bytes/sec for shard deployments 9 | #node.shard.deploy.throttle= 10 | 11 | 12 | ### ~~~~~~~~~~~~~~~~~~~~~~~ ### 13 | ### Lucene related settings ### 14 | ### ~~~~~~~~~~~~~~~~~~~~~~~ ### 15 | 16 | # the percentage (value between 0 and 1) of the client-timeout which should be used 17 | # as server-timeout (as restriction time for the hit-collector). 18 | # Disable the timeout setting the value to 0. 19 | lucene.collector.timeout-percentage=0.75 20 | lucene.collector.track-doc-scores-on-field-sort=false 21 | 22 | lucene.searcher.factory-class=net.sf.katta.lib.lucene.DefaultSearcherFactory 23 | lucene.searcher.threadpool.core-size=25 24 | lucene.searcher.threadpool.max-size=100 25 | 26 | lucene.filter.cache.enabled=true 27 | -------------------------------------------------------------------------------- /modules/katta-core/conf/katta.zk.properties: -------------------------------------------------------------------------------- 1 | # Starts zookeeper embedded in the master jvm. 2 | # We suggest you run zookeeper standalone in large installations 3 | # See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperAdmin.html 4 | zookeeper.embedded=true 5 | 6 | # Comma separated list of host:port of zookeeper servers used by the zookeeper clients. 7 | # Make sure the port is accessible from all nodes and use hostnames and not ip addresses. 8 | # If embedded is set to true, the server hostname is compared to the servers list. 9 | # If found an embedded zookeeper server is started within the master or secondary master jvm. 10 | zookeeper.servers=localhost:2181 11 | 12 | 13 | # The root zk node. This changes with every new cluster. 14 | zookeeper.root-path=/katta 15 | #zookeeper client timeout in milliseconds 16 | zookeeper.timeout=1000 17 | #zookeeper tick time 18 | zookeeper.tick-time=2000 19 | # zookeeper init time limit 20 | zookeeper.init-limit=5 21 | # zookeeper sync limit 22 | zookeeper.sync-limit=2 23 | # zookeeper folder where data are stored 24 | zookeeper.data-dir=./zookeeper-data 25 | # zookeeper folder where log data are stored 26 | zookeeper.log-data-dir=./zookeeper-log-data 27 | 28 | -------------------------------------------------------------------------------- /modules/katta-core/conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | katta.root.logger=INFO,console 3 | log4j.logger.net.sf.katta=INFO 4 | log4j.logger.org.apache.zookeeper=ERROR 5 | log4j.logger.org.apache.hadoop=WARN 6 | log4j.logger.org.I0Itec.zkclient=WARN 7 | katta.log.dir=./logs 8 | katta.log.file=katta.log 9 | 10 | 11 | # Define the root logger to the system property "katta.root.logger". 12 | log4j.rootLogger=${katta.root.logger} 13 | 14 | # Logging Threshold 15 | log4j.threshhold=ALL 16 | 17 | # 18 | # Daily Rolling File Appender 19 | # 20 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 21 | log4j.appender.DRFA.File=${katta.log.dir}/${katta.log.file} 22 | 23 | # Rollver at midnight 24 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 25 | 26 | # 30-day backup 27 | #log4j.appender.DRFA.MaxBackupIndex=30 28 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 29 | 30 | # Pattern format: Date LogLevel LoggerName LogMessage 31 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c:%L - %m%n 32 | 33 | # Debugging Pattern format 34 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 35 | 36 | 37 | # 38 | # console 39 | # Add "console" to rootlogger above if you want to use this 40 | # 41 | log4j.appender.console=org.apache.log4j.ConsoleAppender 42 | log4j.appender.console.target=System.out 43 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 44 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}:%L - %m%n 45 | 46 | # Custom Logging levels 47 | 48 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 49 | -------------------------------------------------------------------------------- /modules/katta-core/conf/nodes: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/commons-cli-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/commons-cli-1.2.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/commons-math-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/commons-math-1.2.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/guava-11.0.1-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/guava-11.0.1-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/guava-11.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/guava-11.0.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/commons-codec-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/commons-codec-1.3.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/commons-httpclient-3.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/commons-httpclient-3.0.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/commons-logging-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/commons-logging-1.0.4.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/commons-logging-api-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/commons-logging-api-1.0.4.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/commons-net-1.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/commons-net-1.4.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/hadoop-0.20.2-core.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/hadoop-0.20.2-core.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/jets3t-0.6.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/jets3t-0.6.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/hadoop/xmlenc-0.52.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/hadoop/xmlenc-0.52.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/jetty-6.1.14.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/jetty-6.1.14.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/jetty-util-6.1.14.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/jetty-util-6.1.14.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/jsch-0.1.41.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/jsch-0.1.41.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/jsp-2.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/jsp-2.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/jsp-api-2.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/jsp-api-2.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/log4j-1.2.15.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/log4j-1.2.15.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/lucene-core-3.5.0-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/lucene-core-3.5.0-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/lucene-core-3.5.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/lucene-core-3.5.0.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/servlet-api-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/servlet-api-2.4.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/typica-1.5.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/typica-1.5.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/zkclient-0.2-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/zkclient-0.2-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/zkclient-0.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/zkclient-0.2.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/compile/zookeeper-3.3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/compile/zookeeper-3.3.2.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/fest-assert-1.3-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/fest-assert-1.3-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/fest-assert-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/fest-assert-1.3.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/fest-util-1.1.4-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/fest-util-1.1.4-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/fest-util-1.1.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/fest-util-1.1.4.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/hadoop-0.20.2-test.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/hadoop-0.20.2-test.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/hamcrest-core-1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/hamcrest-core-1.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/hamcrest-library-1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/hamcrest-library-1.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/junit-4.10-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/junit-4.10-sources.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/junit-4.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/junit-4.10.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/mockito-core-1.8.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/mockito-core-1.8.1.jar -------------------------------------------------------------------------------- /modules/katta-core/lib/test/objenesis-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/lib/test/objenesis-1.0.jar -------------------------------------------------------------------------------- /modules/katta-core/src/it/java/net/sf/katta/integrationTest/DeployPolicyIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.integrationTest; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | 20 | import java.io.File; 21 | import java.util.List; 22 | 23 | import net.sf.katta.integrationTest.support.AbstractIntegrationTest; 24 | import net.sf.katta.node.Node; 25 | import net.sf.katta.protocol.InteractionProtocol; 26 | import net.sf.katta.tool.SampleIndexGenerator; 27 | import net.sf.katta.util.FileUtil; 28 | 29 | import org.junit.Test; 30 | 31 | public class DeployPolicyIntegrationTest extends AbstractIntegrationTest { 32 | 33 | File _indexWithOneShard; 34 | 35 | public DeployPolicyIntegrationTest() { 36 | super(2); 37 | } 38 | 39 | @Override 40 | protected void afterClusterStart() throws Exception { 41 | _indexWithOneShard = new File("build/testdir/oneShardedIndex"); 42 | FileUtil.deleteFolder(_indexWithOneShard); 43 | SampleIndexGenerator sampleIndexGenerator = new SampleIndexGenerator(); 44 | sampleIndexGenerator.createIndex(new String[] { "a", "b", "c" }, _indexWithOneShard.getAbsolutePath(), 2, 3); 45 | } 46 | 47 | @Test 48 | public void testEqualDistributionWhenMoreNodesThenShards() throws Exception { 49 | int replicationCount = 1; 50 | _miniCluster.deployTestIndexes(_indexWithOneShard, getNodeCount(), replicationCount); 51 | 52 | final InteractionProtocol protocol = _miniCluster.getProtocol(); 53 | assertEquals(getNodeCount(), protocol.getIndices().size()); 54 | 55 | protocol.showStructure(false); 56 | List nodes = _miniCluster.getNodes(); 57 | for (Node node : nodes) { 58 | assertEquals(1, node.getContext().getContentServer().getShards().size()); 59 | } 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /modules/katta-core/src/it/java/net/sf/katta/integrationTest/lib/lucene/Hit_MatchLuceneTest.java: -------------------------------------------------------------------------------- 1 | package net.sf.katta.integrationTest.lib.lucene; 2 | 3 | import java.io.IOException; 4 | 5 | import net.sf.katta.lib.lucene.Hit; 6 | import net.sf.katta.lib.lucene.HitTest; 7 | 8 | import org.apache.lucene.search.Scorer; 9 | import org.apache.lucene.search.TopDocs; 10 | import org.apache.lucene.search.TopScoreDocCollector; 11 | import org.junit.Test; 12 | 13 | import static org.junit.Assert.assertEquals; 14 | import static org.mockito.Mockito.mock; 15 | import static org.mockito.Mockito.when; 16 | 17 | /** 18 | * Validates that Hit's sorting behavior matches Lucene's sorting behavior 19 | */ 20 | public class Hit_MatchLuceneTest { 21 | @Test 22 | public void test() throws IOException { 23 | for (HitTest.HitPair hitPair : HitTest.HIT_PAIRS) { 24 | testHitPair(hitPair); 25 | } 26 | } 27 | 28 | public void testHitPair(HitTest.HitPair hitPair) throws IOException { 29 | Scorer scorer = mock(Scorer.class); 30 | when(scorer.score()).thenReturn(hitPair.hitA.getScore()).thenReturn(hitPair.hitB.getScore()); 31 | 32 | TopScoreDocCollector docCollector = TopScoreDocCollector.create(1, false); 33 | docCollector.setScorer(scorer); 34 | docCollector.collect(hitPair.hitA.getDocId()); 35 | docCollector.collect(hitPair.hitB.getDocId()); 36 | 37 | TopDocs topDocs = docCollector.topDocs(); 38 | Hit expectedHit; 39 | if (hitPair.compareValue == -1) { 40 | expectedHit = hitPair.hitA; 41 | } else { 42 | expectedHit = hitPair.hitB; 43 | } 44 | 45 | assertEquals(expectedHit.getScore(), topDocs.scoreDocs[0].score, 0); 46 | assertEquals(expectedHit.getDocId(), topDocs.scoreDocs[0].doc); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/DefaultNameSpaceImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta; 17 | 18 | import net.sf.katta.util.ZkConfiguration; 19 | import net.sf.katta.util.ZkConfiguration.PathDef; 20 | 21 | import org.I0Itec.zkclient.IDefaultNameSpace; 22 | import org.I0Itec.zkclient.ZkClient; 23 | import org.I0Itec.zkclient.exception.ZkNodeExistsException; 24 | import org.apache.log4j.Logger; 25 | 26 | /** 27 | * Implements the default name space in zookeeper for this katta instance. 28 | */ 29 | public class DefaultNameSpaceImpl implements IDefaultNameSpace { 30 | 31 | private static final Logger LOG = Logger.getLogger(DefaultNameSpaceImpl.class); 32 | 33 | private ZkConfiguration _conf; 34 | 35 | public DefaultNameSpaceImpl(ZkConfiguration conf) { 36 | _conf = conf; 37 | } 38 | 39 | @Override 40 | public void createDefaultNameSpace(ZkClient zkClient) { 41 | LOG.debug("Creating default File structure if required...."); 42 | safeCreate(zkClient, _conf.getZkRootPath()); 43 | PathDef[] values = PathDef.values(); 44 | for (PathDef pathDef : values) { 45 | if (pathDef != PathDef.MASTER && pathDef != PathDef.VERSION) { 46 | safeCreate(zkClient, _conf.getZkPath(pathDef)); 47 | } 48 | } 49 | } 50 | 51 | private void safeCreate(ZkClient zkClient, String path) { 52 | try { 53 | // first create parent directories 54 | String parent =ZkConfiguration.getZkParent(path); 55 | if (parent != null && !zkClient.exists(parent)) { 56 | safeCreate(zkClient, parent); 57 | } 58 | 59 | zkClient.createPersistent(path); 60 | } catch (ZkNodeExistsException e) { 61 | // Ignore if the node already exists. 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/IDeployClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.protocol.metadata.IndexMetaData; 21 | 22 | public interface IDeployClient { 23 | 24 | IIndexDeployFuture addIndex(final String name, final String path, final int replicationLevel); 25 | 26 | void removeIndex(final String name); 27 | 28 | boolean existsIndex(String name); 29 | 30 | IndexMetaData getIndexMetaData(String name); 31 | 32 | List getIndices(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/IIndexDeployFuture.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | /** 19 | * 20 | * Future for an index deployment. 21 | * 22 | * @see Future for concept of a future 23 | */ 24 | public interface IIndexDeployFuture { 25 | 26 | /** 27 | * This method blocks until the index has been successfully deployed or the 28 | * deployment failed. 29 | */ 30 | IndexState joinDeployment() throws InterruptedException; 31 | 32 | /** 33 | * This method blocks until the index has been successfully deployed or the 34 | * deployment failed or maxWaitMillis has exceeded. 35 | */ 36 | IndexState joinDeployment(long maxWaitMillis) throws InterruptedException; 37 | 38 | IndexState getState(); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/INodeExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | /** 22 | * How a NodeInteraction resubmits jobs to the WorkQueue to retry failed nodes. 23 | */ 24 | interface INodeExecutor { 25 | 26 | /** 27 | * Submit a new job, which will result in a new NodeInteraction. This is used 28 | * to resubmit jobs to the WorkQueue, to try to get results for failed shards 29 | * an alternate nodes. 30 | * 31 | * @param node 32 | * The node to call. 33 | * @param nodeShardMap 34 | * The entire node to shard map to use. This may have been reduced 35 | * one or two times from the original version from the Client, 36 | * depending on errors and errors on retries. 37 | * @param tryCount 38 | * This job would be the Nth retry. Starts at 1. The NodeInteraction 39 | * uses this to decide whether or not to retry (M tries max). 40 | * @param maxTryCount 41 | * The maximum nuber of retries for a given interaction. 42 | */ 43 | public void execute(String node, Map> nodeShardMap, int tryCount, int maxTryCount); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/IResultPolicy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | /** 19 | * Allows user to get results immediately or wait for more results as they see fit. 20 | * Also specifies if the broadcast call should be terminated and the result closed. 21 | */ 22 | public interface IResultPolicy { 23 | 24 | /** 25 | * How much longer, if any, should we wait for results to arrive. 26 | * Also, should we shutdown the WorkQueue and close the ClientResult? 27 | * 28 | * @param result The results we have so far. 29 | * @return if > 0, sleep at most that many msec, or until a new result 30 | * arrives, whichever comes first. Then call this method again. 31 | * If 0, return the result immediately. 32 | * if < 0, shutdown the WorkQueue, close the result, and return it immediately. 33 | */ 34 | public long waitTime(ClientResult result); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/IResultReceiver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | import java.util.Collection; 19 | 20 | /** 21 | * These are the only ClientResult methods NodeInteraction is allowed to call. 22 | */ 23 | public interface IResultReceiver { 24 | 25 | /** 26 | * @return true if the result is closed, and therefore not accepting any new 27 | * results. 28 | */ 29 | public boolean isClosed(); 30 | 31 | /** 32 | * Add the shard's results. Silently fails if result is closed. 33 | * 34 | * @param result 35 | * The result to add. 36 | * @param shards 37 | * The shards that were called to produce the result. 38 | */ 39 | public void addResult(T result, Collection shards); 40 | 41 | /** 42 | * Report an error thrown by the node when we tried to access the specified 43 | * shards. Silently fails if result is closed. 44 | * 45 | * @param result 46 | * The result to add. 47 | * @param shards 48 | * The shards that were called to produce the result. 49 | */ 50 | public void addError(Throwable error, Collection shards); 51 | 52 | } 53 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/IndexState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | public enum IndexState { 19 | 20 | DEPLOYED, ERROR, DEPLOYING; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/ShardAccessException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | import net.sf.katta.util.KattaException; 19 | 20 | public class ShardAccessException extends KattaException { 21 | 22 | private static final long serialVersionUID = 1L; 23 | 24 | public ShardAccessException(String shard) { 25 | super("Shard '" + shard + "' is currently not reachable"); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/client/ShuffleNodeSelectionPolicy.java: -------------------------------------------------------------------------------- 1 | package net.sf.katta.client; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.Collections; 6 | import java.util.List; 7 | 8 | /** 9 | * Node selection policy that shuffles the node list when updating, thus 10 | * ensuring multiple clients won't connect to the nodes in the same order. 11 | */ 12 | public class ShuffleNodeSelectionPolicy extends BasicNodeSelectionPolicy { 13 | @Override 14 | public void update(String shard, Collection nodes) { 15 | List nodesShuffled = new ArrayList(nodes); 16 | Collections.shuffle(nodesShuffled); 17 | super.update(shard, nodesShuffled); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/lib/lucene/DefaultSearcherFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.lucene; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | 21 | import net.sf.katta.util.NodeConfiguration; 22 | 23 | import org.apache.lucene.search.IndexSearcher; 24 | import org.apache.lucene.store.FSDirectory; 25 | 26 | public class DefaultSearcherFactory implements ISeacherFactory { 27 | 28 | @Override 29 | public void init(NodeConfiguration config) { 30 | // nothing todo 31 | } 32 | 33 | @Override 34 | public IndexSearcher createSearcher(String shardName, File shardDir) throws IOException { 35 | return new IndexSearcher(FSDirectory.open(shardDir.getAbsoluteFile())); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/lib/lucene/ISeacherFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.lucene; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | 21 | import net.sf.katta.util.NodeConfiguration; 22 | 23 | import org.apache.lucene.search.IndexSearcher; 24 | 25 | /** 26 | * A factory for creating {@link IndexSearcher} on a given shard. 27 | * Implementations need to have a default constructor. 28 | */ 29 | public interface ISeacherFactory { 30 | 31 | void init(NodeConfiguration config); 32 | 33 | IndexSearcher createSearcher(String shardName, File shardDir) throws IOException; 34 | } 35 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/lib/mapfile/IMapFileClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.mapfile; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.util.KattaException; 21 | 22 | /** 23 | * The public interface to the front end of the MapFile server. 24 | */ 25 | public interface IMapFileClient { 26 | 27 | /** 28 | * Get all entries with the given key. 29 | * 30 | * @param key The entry(s) to look up. 31 | * @param indexNames The MapFiles to search. 32 | * @return All the entries with the given key. 33 | * @throws KattaException 34 | */ 35 | public List get(String key, final String[] indexNames) throws KattaException; 36 | 37 | /** 38 | * Closes down the client. 39 | */ 40 | public void close(); 41 | 42 | } -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/lib/mapfile/IMapFileServer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.mapfile; 17 | 18 | import java.io.IOException; 19 | 20 | 21 | import org.apache.hadoop.io.Text; 22 | import org.apache.hadoop.ipc.VersionedProtocol; 23 | 24 | /** 25 | * Interface for the client calls that will arrive via Hadoop RPC. 26 | * 27 | * This server looks up Text entries from MapFiles using Text keys. 28 | */ 29 | public interface IMapFileServer extends VersionedProtocol { 30 | 31 | /** 32 | * Get all the occurrences of the given Text key. There could be 33 | * up to one entry per shard. 34 | * 35 | * @param key The key to search for. 36 | * @param shards Which MapFile shards to look in. 37 | * @return The list of Text results. 38 | * @throws IOException If an error occurs. 39 | */ 40 | public TextArrayWritable get(Text key, String[] shards) throws IOException; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/lib/mapfile/TextArrayWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.mapfile; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | import java.util.Collections; 22 | import java.util.List; 23 | 24 | import org.apache.hadoop.io.ArrayWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.io.Writable; 27 | 28 | /** 29 | * This class provides a way to return a list of Text objects via Hadoop RPC. It 30 | * provides a zero-arg constructor, which Hadoop RPC requires for return types. 31 | */ 32 | public class TextArrayWritable implements Writable { 33 | 34 | public ArrayWritable array; 35 | 36 | @SuppressWarnings("unchecked") 37 | public TextArrayWritable() { 38 | this(Collections.EMPTY_LIST); 39 | } 40 | 41 | public TextArrayWritable(List texts) { 42 | array = new ArrayWritable(Text.class, texts.toArray(new Writable[texts.size()])); 43 | } 44 | 45 | public void readFields(DataInput in) throws IOException { 46 | array.readFields(in); 47 | } 48 | 49 | public void write(DataOutput out) throws IOException { 50 | array.write(out); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/master/IDeployPolicy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.master; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | /** 22 | * Exchangeable policy for which creates an distribution plan for the shards of 23 | * one index. 24 | * 25 | */ 26 | public interface IDeployPolicy { 27 | 28 | /** 29 | * Creates a distribution plan for the shards of one index. Note that the 30 | * index can already be deployed, in that case its more a "replication" plan. 31 | * 32 | * @param currentShard2NodesMap 33 | * all current deployments of the shards of the one index to 34 | * distribute/replicate 35 | * @param currentNode2ShardsMap 36 | * all nodes and their shards 37 | * @param aliveNodes 38 | * @param replicationLevel 39 | * @return 40 | */ 41 | Map> createDistributionPlan(Map> currentShard2NodesMap, 42 | Map> currentNode2ShardsMap, List aliveNodes, int replicationLevel); 43 | 44 | } 45 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/master/MasterContext.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.master; 17 | 18 | import java.io.IOException; 19 | 20 | import net.sf.katta.protocol.InteractionProtocol; 21 | import net.sf.katta.protocol.MasterQueue; 22 | import net.sf.katta.protocol.metadata.IndexMetaData; 23 | import net.sf.katta.util.HadoopUtil; 24 | 25 | import org.apache.hadoop.fs.FileSystem; 26 | import org.apache.hadoop.fs.Path; 27 | 28 | public class MasterContext { 29 | 30 | private final Master _master; 31 | private final InteractionProtocol _protocol; 32 | private final IDeployPolicy _deployPolicy; 33 | private final MasterQueue _masterQueue; 34 | 35 | public MasterContext(InteractionProtocol protocol, Master master, IDeployPolicy deployPolicy, MasterQueue masterQueue) { 36 | _protocol = protocol; 37 | _master = master; 38 | _deployPolicy = deployPolicy; 39 | _masterQueue = masterQueue; 40 | } 41 | 42 | public InteractionProtocol getProtocol() { 43 | return _protocol; 44 | } 45 | 46 | public Master getMaster() { 47 | return _master; 48 | } 49 | 50 | public IDeployPolicy getDeployPolicy() { 51 | return _deployPolicy; 52 | } 53 | 54 | public MasterQueue getMasterQueue() { 55 | return _masterQueue; 56 | } 57 | 58 | public FileSystem getFileSystem(IndexMetaData indexMd) throws IOException { 59 | return HadoopUtil.getFileSystem(new Path(indexMd.getPath())); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/node/NodeContext.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.node; 17 | 18 | import net.sf.katta.protocol.InteractionProtocol; 19 | 20 | public class NodeContext { 21 | 22 | private final Node _node; 23 | private final ShardManager _shardManager; 24 | private final InteractionProtocol _protocol; 25 | private final IContentServer _nodeManaged; 26 | 27 | public NodeContext(InteractionProtocol protocol, Node node, ShardManager shardManager, IContentServer nodeManaged) { 28 | _protocol = protocol; 29 | _node = node; 30 | _shardManager = shardManager; 31 | _nodeManaged = nodeManaged; 32 | } 33 | 34 | public Node getNode() { 35 | return _node; 36 | } 37 | 38 | public ShardManager getShardManager() { 39 | return _shardManager; 40 | } 41 | 42 | public InteractionProtocol getProtocol() { 43 | return _protocol; 44 | } 45 | 46 | public IContentServer getContentServer() { 47 | return _nodeManaged; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/node/monitor/IMonitor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.node.monitor; 17 | 18 | import net.sf.katta.protocol.InteractionProtocol; 19 | 20 | public interface IMonitor { 21 | 22 | void startMonitoring(String serverId, InteractionProtocol protocol); 23 | 24 | public void stopMonitoring(); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/node/monitor/MetricsRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.node.monitor; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class MetricsRecord implements Serializable { 23 | private static final long serialVersionUID = 1832158671826263268L; 24 | 25 | private List _records = new ArrayList(); 26 | 27 | private final String _serverId; 28 | 29 | public MetricsRecord(String serverId) { 30 | _serverId = serverId; 31 | } 32 | 33 | public String getServerId() { 34 | return _serverId; 35 | } 36 | 37 | public void addValue(String key, long value, long timeStamp) { 38 | _records.add(new Record(key, value, timeStamp)); 39 | } 40 | 41 | public List getRecords() { 42 | return _records; 43 | } 44 | 45 | public class Record implements Serializable { 46 | 47 | private static final long serialVersionUID = 8134090886018804896L; 48 | 49 | private final String _key; 50 | private final long _value; 51 | private final long _timeStamp; 52 | 53 | public Record(String key, long value, long timeStamp) { 54 | _key = key; 55 | _value = value; 56 | _timeStamp = timeStamp; 57 | } 58 | 59 | public String getKey() { 60 | return _key; 61 | } 62 | 63 | public long getValue() { 64 | return _value; 65 | } 66 | 67 | public long getTimeStamp() { 68 | return _timeStamp; 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return _key + " " + _value + " " + _timeStamp; 74 | } 75 | } 76 | 77 | @Override 78 | public String toString() { 79 | return _serverId + ": " + _records; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/OperationId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation; 17 | 18 | import java.io.Serializable; 19 | 20 | public class OperationId implements Serializable { 21 | 22 | private static final long serialVersionUID = 1L; 23 | private final String _nodeName; 24 | private final String _elementName; 25 | 26 | public OperationId(String nodeName, String elementName) { 27 | _nodeName = nodeName; 28 | _elementName = elementName; 29 | } 30 | 31 | public String getNodeName() { 32 | return _nodeName; 33 | } 34 | 35 | public String getElementName() { 36 | return _elementName; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return _nodeName + "-" + _elementName; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/master/IndexDeployException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.master; 17 | 18 | import net.sf.katta.protocol.metadata.IndexDeployError.ErrorType; 19 | 20 | public class IndexDeployException extends Exception { 21 | 22 | private static final long serialVersionUID = 1L; 23 | private final ErrorType _errorType; 24 | 25 | public IndexDeployException(ErrorType errorType, final String message) { 26 | super(message); 27 | _errorType = errorType; 28 | } 29 | 30 | public IndexDeployException(ErrorType errorType, final String message, final Throwable cause) { 31 | super(message, cause); 32 | _errorType = errorType; 33 | } 34 | 35 | public ErrorType getErrorType() { 36 | return _errorType; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/master/IndexReinitializeOperation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.master; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.master.MasterContext; 21 | import net.sf.katta.operation.OperationId; 22 | import net.sf.katta.operation.node.OperationResult; 23 | import net.sf.katta.protocol.InteractionProtocol; 24 | import net.sf.katta.protocol.metadata.IndexMetaData; 25 | 26 | import org.I0Itec.zkclient.ExceptionUtil; 27 | 28 | @SuppressWarnings("serial") 29 | public class IndexReinitializeOperation extends IndexDeployOperation { 30 | 31 | public IndexReinitializeOperation(IndexMetaData indexMD) { 32 | super(indexMD.getName(), indexMD.getPath(), indexMD.getReplicationLevel()); 33 | } 34 | 35 | @Override 36 | public List execute(MasterContext context, List runningOperations) throws Exception { 37 | InteractionProtocol protocol = context.getProtocol(); 38 | try { 39 | _indexMD.getShards().addAll(readShardsFromFs(_indexMD.getName(), _indexMD.getPath())); 40 | protocol.updateIndexMD(_indexMD); 41 | } catch (Exception e) { 42 | ExceptionUtil.rethrowInterruptedException(e); 43 | handleMasterDeployException(protocol, _indexMD, e); 44 | } 45 | return null; 46 | } 47 | 48 | @Override 49 | public ExecutionInstruction getExecutionInstruction(List runningOperations) throws Exception { 50 | return ExecutionInstruction.EXECUTE; 51 | } 52 | 53 | @Override 54 | public void nodeOperationsComplete(MasterContext context, List results) throws Exception { 55 | // nothing todo 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/DeployResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | public class DeployResult extends OperationResult { 22 | 23 | private static final long serialVersionUID = 1L; 24 | 25 | private Map _exceptionByShard = new HashMap(3); 26 | private Map> _metadataMapByShard = new HashMap>(3); 27 | 28 | public DeployResult(String nodeName) { 29 | super(nodeName); 30 | } 31 | 32 | public void addShardException(String shardName, Exception exception) { 33 | _exceptionByShard.put(shardName, exception); 34 | } 35 | 36 | public void addShardMetaDataMap(String shardName, Map shardMetaData) { 37 | _metadataMapByShard.put(shardName, shardMetaData); 38 | } 39 | 40 | public Map getShardExceptions() { 41 | return _exceptionByShard; 42 | } 43 | 44 | public Map> getShardMetaDataMaps() { 45 | return _metadataMapByShard; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/NodeOperation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.io.Serializable; 19 | 20 | import net.sf.katta.node.NodeContext; 21 | 22 | /** 23 | * An operation executed by an {@link Node}. 24 | */ 25 | public interface NodeOperation extends Serializable { 26 | 27 | /** 28 | * @param context 29 | * @return null or an {@link OperationResult} 30 | * @throws InterruptedException 31 | */ 32 | OperationResult execute(NodeContext context) throws InterruptedException; 33 | } 34 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/OperationResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.io.Serializable; 19 | 20 | public class OperationResult implements Serializable { 21 | 22 | private static final long serialVersionUID = 1L; 23 | 24 | private final String _nodeName; 25 | private final Exception _unhandledException; 26 | 27 | public OperationResult(String nodeName) { 28 | this(nodeName, null); 29 | } 30 | 31 | public OperationResult(String nodeName, Exception unhandledException) { 32 | _nodeName = nodeName; 33 | _unhandledException = unhandledException; 34 | } 35 | 36 | public String getNodeName() { 37 | return _nodeName; 38 | } 39 | 40 | public Exception getUnhandledException() { 41 | return _unhandledException; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/ShardDeployOperation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.io.File; 19 | import java.util.Map; 20 | 21 | import net.sf.katta.node.IContentServer; 22 | import net.sf.katta.node.NodeContext; 23 | 24 | public class ShardDeployOperation extends AbstractShardOperation { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | @Override 29 | protected String getOperationName() { 30 | return "deploy"; 31 | } 32 | 33 | @Override 34 | protected void execute(NodeContext context, String shardName, DeployResult deployResult) throws Exception { 35 | String shardPath = getShardPath(shardName); 36 | File localShardFolder = context.getShardManager().installShard(shardName, shardPath); 37 | IContentServer contentServer = context.getContentServer(); 38 | if (!contentServer.getShards().contains(shardName)) { 39 | contentServer.addShard(shardName, localShardFolder); 40 | Map shardMetaData = context.getContentServer().getShardMetaData(shardName); 41 | if (shardMetaData == null) { 42 | throw new IllegalStateException("node managed '" + context.getContentServer() 43 | + "' does return NULL as shard metadata"); 44 | } 45 | deployResult.addShardMetaDataMap(shardName, shardMetaData); 46 | } 47 | publishShard(shardName, context); 48 | } 49 | 50 | @Override 51 | protected void onException(NodeContext context, String shardName, Exception e) { 52 | context.getShardManager().uninstallShard(shardName); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/ShardRedeployOperation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.io.File; 19 | import java.util.Collection; 20 | 21 | import net.sf.katta.node.IContentServer; 22 | import net.sf.katta.node.NodeContext; 23 | 24 | /** 25 | * Redploys shards which are already installed in {@link ShardManager}. 26 | */ 27 | public class ShardRedeployOperation extends AbstractShardOperation { 28 | 29 | private static final long serialVersionUID = 1L; 30 | 31 | public ShardRedeployOperation(Collection installedShards) { 32 | for (String shardName : installedShards) { 33 | addShard(shardName); 34 | } 35 | } 36 | 37 | @Override 38 | protected String getOperationName() { 39 | return "redeploy"; 40 | } 41 | 42 | @Override 43 | protected void execute(NodeContext context, String shardName, DeployResult deployResult) throws Exception { 44 | File localShardFolder = context.getShardManager().getShardFolder(shardName); 45 | IContentServer contentServer = context.getContentServer(); 46 | if (!contentServer.getShards().contains(shardName)) { 47 | contentServer.addShard(shardName, localShardFolder); 48 | } 49 | publishShard(shardName, context); 50 | } 51 | 52 | @Override 53 | protected void onException(NodeContext context, String shardName, Exception e) { 54 | context.getShardManager().uninstallShard(shardName); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/operation/node/ShardUndeployOperation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.node.NodeContext; 21 | 22 | public class ShardUndeployOperation extends AbstractShardOperation { 23 | 24 | private static final long serialVersionUID = 1L; 25 | 26 | public ShardUndeployOperation(List shards) { 27 | for (String shard : shards) { 28 | addShard(shard); 29 | } 30 | } 31 | 32 | @Override 33 | protected String getOperationName() { 34 | return "undeploy"; 35 | } 36 | 37 | @Override 38 | protected void execute(NodeContext context, String shardName, DeployResult deployResult) throws Exception { 39 | context.getProtocol().unpublishShard(context.getNode(), shardName); 40 | context.getContentServer().removeShard(shardName); 41 | context.getShardManager().uninstallShard(shardName); 42 | } 43 | 44 | @Override 45 | protected void onException(NodeContext context, String shardName, Exception e) { 46 | context.getShardManager().uninstallShard(shardName); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/ConnectedComponent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol; 17 | 18 | /** 19 | * A component which is connected to a node-cluster / zk-server. But sometimes 20 | * connections drop. So this is the lifecycle for connections (once they are 21 | * already connected). 22 | * 23 | */ 24 | public interface ConnectedComponent { 25 | 26 | void reconnect(); 27 | 28 | void disconnect(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/IAddRemoveListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol; 17 | 18 | /** 19 | * Registerable with {@link InteractionProtocol}. 20 | * 21 | * @see InteractionProtocol#registerIndexListener(ConnectedComponent, 22 | * IndexListener) 23 | * @see InteractionProtocol#registerNodeListener(ConnectedComponent, 24 | * NodeListener) 25 | */ 26 | public interface IAddRemoveListener { 27 | 28 | void added(String name); 29 | 30 | void removed(String name); 31 | } 32 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/metadata/IndexDeployError.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol.metadata; 17 | 18 | import java.io.Serializable; 19 | import java.util.List; 20 | 21 | import net.sf.katta.util.One2ManyListMap; 22 | 23 | import com.google.common.base.Objects; 24 | 25 | public class IndexDeployError implements Serializable { 26 | 27 | private static final long serialVersionUID = 1L; 28 | 29 | public static enum ErrorType { 30 | NO_NODES_AVAILIBLE, INDEX_NOT_ACCESSIBLE, SHARDS_NOT_DEPLOYABLE, UNKNOWN; 31 | } 32 | 33 | private final String _indexName; 34 | private final ErrorType _errorType; 35 | private final One2ManyListMap _shard2ExceptionsMap = new One2ManyListMap(); 36 | private Exception _exception; 37 | 38 | public IndexDeployError(String indexName, ErrorType errorType) { 39 | _indexName = indexName; 40 | _errorType = errorType; 41 | } 42 | 43 | public String getIndexName() { 44 | return _indexName; 45 | } 46 | 47 | public ErrorType getErrorType() { 48 | return _errorType; 49 | } 50 | 51 | public void setException(Exception exception) { 52 | _exception = exception; 53 | } 54 | 55 | public Exception getException() { 56 | return _exception; 57 | } 58 | 59 | public void addShardError(String shardName, Exception exception) { 60 | _shard2ExceptionsMap.add(shardName, exception); 61 | } 62 | 63 | public List getShardErrors(String shardName) { 64 | return _shard2ExceptionsMap.getValues(shardName); 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | return Objects.toStringHelper(this).addValue(_indexName).addValue(_errorType).addValue(_exception).toString(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/metadata/MasterMetaData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol.metadata; 17 | 18 | import java.io.Serializable; 19 | 20 | import net.sf.katta.util.DefaultDateFormat; 21 | 22 | public class MasterMetaData implements Serializable { 23 | 24 | private static final long serialVersionUID = 1L; 25 | 26 | private String _masterName; 27 | private long _startTime; 28 | 29 | public MasterMetaData(final String masterName, final long startTime) { 30 | _masterName = masterName; 31 | _startTime = startTime; 32 | } 33 | 34 | public String getStartTimeAsString() { 35 | return DefaultDateFormat.longToDateString(_startTime); 36 | } 37 | 38 | public String getMasterName() { 39 | return _masterName; 40 | } 41 | 42 | public long getStartTime() { 43 | return _startTime; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return getMasterName() + ":" + getStartTime(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/metadata/NodeMetaData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol.metadata; 17 | 18 | import java.io.Serializable; 19 | 20 | import net.sf.katta.util.DefaultDateFormat; 21 | 22 | public class NodeMetaData implements Serializable { 23 | 24 | private static final long serialVersionUID = 1L; 25 | 26 | private String _name; 27 | private float _queriesPerMinute = 0f; 28 | private long _startTimeStamp = System.currentTimeMillis(); 29 | 30 | // with node execution 31 | 32 | public NodeMetaData() { 33 | // for serialization 34 | } 35 | 36 | public NodeMetaData(final String name) { 37 | _name = name; 38 | } 39 | 40 | public String getName() { 41 | return _name; 42 | } 43 | 44 | public String getStartTimeAsDate() { 45 | return DefaultDateFormat.longToDateString(_startTimeStamp); 46 | } 47 | 48 | public float getQueriesPerMinute() { 49 | return _queriesPerMinute; 50 | } 51 | 52 | public void setQueriesPerMinute(final float queriesPerMinute) { 53 | _queriesPerMinute = queriesPerMinute; 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | return getName() + "\t:\t" + getStartTimeAsDate() + ""; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/protocol/upgrade/UpgradeAction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.protocol.upgrade; 17 | 18 | import net.sf.katta.protocol.InteractionProtocol; 19 | 20 | public interface UpgradeAction { 21 | 22 | public void upgrade(InteractionProtocol protocol); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/ec2/Ec2Configuration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.ec2; 17 | 18 | import net.sf.katta.util.KattaConfiguration; 19 | 20 | @SuppressWarnings("serial") 21 | public class Ec2Configuration extends KattaConfiguration { 22 | 23 | final static String ACCOUNT_ID = "aws.accountId"; 24 | final static String ACCESS_KEY = "aws.accessKeyId"; 25 | final static String SECRET_ACCESS_KEY = "aws.secretAccessKey"; 26 | final static String KEY_PAIR_NAME = "aws.keyPairName"; 27 | private static final String AIM = "aws.aim"; 28 | 29 | public Ec2Configuration() { 30 | super("/katta.ec2.properties"); 31 | } 32 | 33 | public String getAccountId() { 34 | return getProperty(ACCOUNT_ID); 35 | } 36 | 37 | public String getAccessKey() { 38 | return getProperty(ACCESS_KEY); 39 | } 40 | 41 | public String getSecretAccessKey() { 42 | return getProperty(SECRET_ACCESS_KEY); 43 | } 44 | 45 | public String getKeyName() { 46 | return getProperty(KEY_PAIR_NAME); 47 | } 48 | 49 | public String getKeyPath() { 50 | return getProperty("aws.keyPath"); 51 | } 52 | 53 | public String getAim() { 54 | return getProperty(AIM); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/ec2/Ec2Instance.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.ec2; 17 | 18 | public class Ec2Instance { 19 | private final String _privateDnsName; 20 | private final String _publicDnsName; 21 | 22 | public Ec2Instance(String privateDnsName, String publicdnsName) { 23 | _privateDnsName = privateDnsName; 24 | _publicDnsName = publicdnsName; 25 | } 26 | 27 | public String getInternalHost() { 28 | return _privateDnsName; 29 | } 30 | 31 | public String getExternalHost() { 32 | return _publicDnsName; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/loadtest/LoadTestNodeOperationResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.operation.node.OperationResult; 21 | 22 | @SuppressWarnings("serial") 23 | public class LoadTestNodeOperationResult extends OperationResult { 24 | 25 | private final List _queryResults; 26 | 27 | public LoadTestNodeOperationResult(String nodeName, List queryResults) { 28 | super(nodeName); 29 | _queryResults = queryResults; 30 | } 31 | 32 | public List getQueryResults() { 33 | return _queryResults; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/loadtest/LoadTestQueryResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest; 17 | 18 | import java.io.Serializable; 19 | 20 | @SuppressWarnings("serial") 21 | public class LoadTestQueryResult implements Serializable { 22 | 23 | private long _startTime; 24 | private long _endTime; 25 | private String _query; 26 | private String _nodeId; 27 | 28 | public LoadTestQueryResult() { 29 | // do nothing 30 | } 31 | 32 | public LoadTestQueryResult(long startTime, long endTime, String query, String nodeId) { 33 | _startTime = startTime; 34 | _endTime = endTime; 35 | _query = query; 36 | _nodeId = nodeId; 37 | } 38 | 39 | public long getStartTime() { 40 | return _startTime; 41 | } 42 | 43 | public void setStartTime(long startTime) { 44 | _startTime = startTime; 45 | } 46 | 47 | public long getEndTime() { 48 | return _endTime; 49 | } 50 | 51 | public void setEndTime(long endTime) { 52 | _endTime = endTime; 53 | } 54 | 55 | public String getQuery() { 56 | return _query; 57 | } 58 | 59 | public void setQuery(String query) { 60 | _query = query; 61 | } 62 | 63 | public String getNodeId() { 64 | return _nodeId; 65 | } 66 | 67 | public void setNodeId(String nodeId) { 68 | _nodeId = nodeId; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/loadtest/query/AbstractQueryExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest.query; 17 | 18 | import java.io.Serializable; 19 | 20 | import net.sf.katta.node.NodeContext; 21 | 22 | @SuppressWarnings("serial") 23 | public abstract class AbstractQueryExecutor implements Serializable { 24 | 25 | protected final String[] _indices; 26 | protected final String[] _queries; 27 | 28 | public AbstractQueryExecutor(String[] indices, String[] queries) { 29 | _indices = indices; 30 | _queries = queries; 31 | } 32 | 33 | public String[] getQueries() { 34 | return _queries; 35 | } 36 | 37 | public String[] getIndices() { 38 | return _indices; 39 | } 40 | 41 | /** 42 | * Called from the loadtest node before calling 43 | * {@link #execute(NodeContext, String)} method. 44 | * 45 | * @param nodeContext 46 | * @param zkConf 47 | * the configuration of the target cluster 48 | * @throws Exception 49 | */ 50 | public abstract void init(NodeContext nodeContext) throws Exception; 51 | 52 | /** 53 | * Called from the loadtest node after calling 54 | * {@link #execute(NodeContext, String)} method the last time. 55 | * 56 | * @param nodeContext 57 | * @throws Exception 58 | */ 59 | public abstract void close(NodeContext nodeContext) throws Exception; 60 | 61 | /** 62 | * Might called multiple times from the loadtest node, depending on query 63 | * rate. 64 | * 65 | * @param nodeContext 66 | * @param query 67 | * @throws Exception 68 | */ 69 | public abstract void execute(NodeContext nodeContext, String query) throws Exception; 70 | 71 | } 72 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/loadtest/query/LuceneSearchExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest.query; 17 | 18 | import net.sf.katta.lib.lucene.ILuceneClient; 19 | import net.sf.katta.lib.lucene.LuceneClient; 20 | import net.sf.katta.node.NodeContext; 21 | import net.sf.katta.util.ZkConfiguration; 22 | 23 | import org.apache.lucene.analysis.KeywordAnalyzer; 24 | import org.apache.lucene.queryParser.QueryParser; 25 | import org.apache.lucene.search.Query; 26 | import org.apache.lucene.util.Version; 27 | 28 | @SuppressWarnings("serial") 29 | public class LuceneSearchExecutor extends AbstractQueryExecutor { 30 | 31 | private final int _count; 32 | private ILuceneClient _client; 33 | private final ZkConfiguration _zkConfOfTargetCluster; 34 | 35 | public LuceneSearchExecutor(String[] indices, String[] queries, ZkConfiguration zkConfOfTargetCluster, int count) { 36 | super(indices, queries); 37 | _zkConfOfTargetCluster = zkConfOfTargetCluster; 38 | _count = count; 39 | } 40 | 41 | @Override 42 | public void init(NodeContext nodeContext) throws Exception { 43 | _client = new LuceneClient(_zkConfOfTargetCluster); 44 | } 45 | 46 | @Override 47 | public void close(NodeContext nodeContext) throws Exception { 48 | _client.close(); 49 | _client = null; 50 | } 51 | 52 | @Override 53 | public void execute(NodeContext nodeContext, String queryString) throws Exception { 54 | final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse(queryString); 55 | _client.search(query, _indices, _count); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/tool/loadtest/query/MapfileAccessExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest.query; 17 | 18 | import net.sf.katta.lib.mapfile.IMapFileClient; 19 | import net.sf.katta.lib.mapfile.MapFileClient; 20 | import net.sf.katta.node.NodeContext; 21 | import net.sf.katta.util.ZkConfiguration; 22 | 23 | @SuppressWarnings("serial") 24 | public class MapfileAccessExecutor extends AbstractQueryExecutor { 25 | 26 | private IMapFileClient _client; 27 | private final ZkConfiguration _zkConfOfTargetCluster; 28 | 29 | public MapfileAccessExecutor(String[] indices, String[] queries, ZkConfiguration zkConfOfTargetCluster) { 30 | super(indices, queries); 31 | _zkConfOfTargetCluster = zkConfOfTargetCluster; 32 | } 33 | 34 | @Override 35 | public void init(NodeContext nodeContext) throws Exception { 36 | _client = new MapFileClient(_zkConfOfTargetCluster); 37 | } 38 | 39 | @Override 40 | public void close(NodeContext nodeContext) throws Exception { 41 | _client.close(); 42 | _client = null; 43 | } 44 | 45 | @Override 46 | public void execute(NodeContext nodeContext, String queryString) throws Exception { 47 | _client.get(queryString, _indices); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/ClientConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | /** 19 | * 20 | * Configuration for a {@link Client}. 21 | * 22 | *

23 | * RPC Configuration
24 | * For the client-node interaction hadoop rpc is used as the underlying 25 | * communication technology.
26 | * To fine tune the communication you can also set hadoop rpc properties (like 27 | * 'ipc.client.connect.max.retries').
28 | * Those properties will be propagated to hadoop. 29 | * 30 | */ 31 | @SuppressWarnings("serial") 32 | public class ClientConfiguration extends KattaConfiguration { 33 | 34 | public final static String CLIENT_NODE_INTERACTION_MAXTRYCOUNT = "client.node.interaction.maxTryCount"; 35 | 36 | public ClientConfiguration() { 37 | super(); 38 | // set default values 39 | setProperty(CLIENT_NODE_INTERACTION_MAXTRYCOUNT, 3); 40 | 41 | // default values for underlying hadoop rpc (used for searching on nodes) 42 | setProperty("ipc.client.connect.max.retries", 2); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/DefaultDateFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.text.SimpleDateFormat; 19 | import java.util.Date; 20 | 21 | public class DefaultDateFormat { 22 | 23 | public final static SimpleDateFormat FORMAT = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); 24 | 25 | public static String longToDateString(final long timeStamp) { 26 | return FORMAT.format(new Date(timeStamp)); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/HadoopUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.hadoop.conf.Configuration; 21 | import org.apache.hadoop.fs.FileSystem; 22 | import org.apache.hadoop.fs.Path; 23 | 24 | public class HadoopUtil { 25 | 26 | public static FileSystem getFileSystem(Path path) throws IOException { 27 | synchronized (FileSystem.class) { 28 | // had once a ConcurrentModificationException 29 | return FileSystem.get(path.toUri(), new Configuration()); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/IHadoopConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | /** 19 | * Contains hadoop configuration keys used in 20 | * hadoop-default.xml/hadoop-site.xml. 21 | */ 22 | public interface IHadoopConstants { 23 | 24 | String NAMENODE = "fs.default.name"; 25 | String JOBTRACKER = "mapred.job.tracker"; 26 | 27 | String TMP_DIR = "hadoop.tmp.dir"; 28 | 29 | String NATIVE_LIB = "hadoop.native.lib"; 30 | 31 | String NUMBER_OF_MAP_TASKS_PER_JOB = "mapred.map.tasks"; 32 | String NUMBER_OF_REDUCE_TASKS_PER_JOB = "mapred.reduce.tasks"; 33 | String NUMBER_OF_MAP_TASKS_PER_TASKTRACKER = "mapred.tasktracker.map.tasks.maximum"; 34 | String NUMBER_OF_REDUCE_TASKS_PER_TASKTRACKER = "mapred.tasktracker.reduce.tasks.maximum"; 35 | String CHILD_JVM_OPTS = "mapred.child.java.opts"; 36 | 37 | String MAPRED_LOCAL_DIR = "mapred.local.dir"; 38 | String MAPRED_SYSTEM_DIR = "mapred.system.dir"; 39 | String MAPRED_TEMP_DIR = "mapred.temp.dir"; 40 | 41 | String IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; 42 | String IO_SORT_MB = "io.sort.mb"; 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/KattaException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | public class KattaException extends Exception { 19 | 20 | private static final long serialVersionUID = 1L; 21 | 22 | public KattaException(String message) { 23 | super(message); 24 | } 25 | 26 | public KattaException(final String msg, final Throwable t) { 27 | super(msg, t); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/MasterConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.io.File; 19 | import java.util.Properties; 20 | 21 | @SuppressWarnings("serial") 22 | public class MasterConfiguration extends KattaConfiguration { 23 | 24 | public final static String DEPLOY_POLICY = "master.deploy.policy"; 25 | public final static String SAFE_MODE_MAX_TIME = "safemode.maxTime"; 26 | 27 | public MasterConfiguration() { 28 | super("/katta.master.properties"); 29 | } 30 | 31 | public MasterConfiguration(File file) { 32 | super(file); 33 | } 34 | 35 | public MasterConfiguration(Properties properties) { 36 | super(properties, null); 37 | } 38 | 39 | public String getDeployPolicy() { 40 | return getProperty(DEPLOY_POLICY, "net.sf.katta.master.DefaultDistributionPolicy"); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/PropertyUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.io.File; 19 | import java.io.FileInputStream; 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | import java.net.URL; 23 | import java.util.Properties; 24 | 25 | public class PropertyUtil { 26 | 27 | public static Properties loadProperties(String path) { 28 | InputStream inputStream = PropertyUtil.class.getResourceAsStream(path); 29 | if (inputStream == null) { 30 | throw new RuntimeException(path + " not in classpath"); 31 | } 32 | final Properties properties = new Properties(); 33 | try { 34 | properties.load(inputStream); 35 | return properties; 36 | } catch (final IOException e) { 37 | throw new RuntimeException("unable to load kata.properties", e); 38 | } 39 | } 40 | 41 | public static String getPropertiesFilePath(final String path) { 42 | URL resource = PropertyUtil.class.getResource(path); 43 | if (resource == null) { 44 | throw new RuntimeException(path + " not in classpath"); 45 | } 46 | return resource.toString(); 47 | } 48 | 49 | public static Properties loadProperties(final File file) { 50 | final Properties properties = new Properties(); 51 | try { 52 | FileInputStream inStream = new FileInputStream(file); 53 | properties.load(inStream); 54 | return properties; 55 | } catch (final IOException e) { 56 | throw new RuntimeException("unable to load kata.properties", e); 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/java/net/sf/katta/util/ZkKattaUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import net.sf.katta.DefaultNameSpaceImpl; 19 | 20 | import org.I0Itec.zkclient.ZkClient; 21 | import org.I0Itec.zkclient.ZkServer; 22 | 23 | public class ZkKattaUtil { 24 | 25 | public static final int DEFAULT_PORT = 2181; 26 | 27 | public static ZkClient startZkClient(ZkConfiguration conf, int connectionTimeout) { 28 | return new ZkClient(conf.getZKServers(), conf.getZKTimeOut(), connectionTimeout); 29 | } 30 | 31 | public static ZkServer startZkServer(ZkConfiguration conf) { 32 | return startZkServer(conf, DEFAULT_PORT); 33 | } 34 | 35 | public static ZkServer startZkServer(ZkConfiguration conf, int port) { 36 | ZkServer zkServer = new ZkServer(conf.getZKDataDir(), conf.getZKDataLogDir(), new DefaultNameSpaceImpl(conf), port, 37 | conf.getZKTickTime()); 38 | zkServer.start(); 39 | return zkServer; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /modules/katta-core/src/main/resources/xpath.properties: -------------------------------------------------------------------------------- 1 | xpath.input.file=file:///tmp/xpath.properties -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/AbstractWritableTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.hadoop.io.DataInputBuffer; 21 | import org.apache.hadoop.io.DataOutputBuffer; 22 | import org.apache.hadoop.io.Writable; 23 | 24 | /** 25 | * Abstract base test for {@link Writable}s. 26 | */ 27 | public abstract class AbstractWritableTest extends AbstractTest { 28 | 29 | protected DataOutputBuffer writeWritable(Writable writable) throws IOException { 30 | DataOutputBuffer out = new DataOutputBuffer(1024); 31 | writable.write(out); 32 | out.flush(); 33 | return out; 34 | } 35 | 36 | protected Writable readWritable(DataOutputBuffer out, Writable writable) throws IOException { 37 | DataInputBuffer inputBuffer = new DataInputBuffer(); 38 | inputBuffer.reset(out.getData(), out.getData().length); 39 | writable.readFields(inputBuffer); 40 | return writable; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/AbstractZkTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta; 17 | 18 | import net.sf.katta.protocol.InteractionProtocol; 19 | import net.sf.katta.testutil.ZkTestSystem; 20 | 21 | import org.junit.Rule; 22 | 23 | public abstract class AbstractZkTest extends AbstractTest { 24 | 25 | @Rule 26 | public ZkTestSystem _zk = ZkTestSystem.getInstance(); 27 | 28 | protected InteractionProtocol _protocol = _zk.getInteractionProtocol(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/KattaTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta; 17 | 18 | import net.sf.katta.Katta.Command; 19 | import net.sf.katta.protocol.metadata.IndexMetaData; 20 | 21 | import org.junit.Test; 22 | 23 | public class KattaTest extends AbstractZkTest { 24 | 25 | @Test 26 | public void testShowStructure() throws Exception { 27 | execute(Katta.SHOW_STRUCTURE_COMMAND, "showStructure"); 28 | execute(Katta.SHOW_STRUCTURE_COMMAND, "showStructure", "-d"); 29 | } 30 | 31 | @Test(expected = Exception.class) 32 | public void testShowErrorsWithIndexNotExist() throws Exception { 33 | execute(Katta.LIST_ERRORS_COMMAND, "showStructure", "a"); 34 | } 35 | 36 | @Test 37 | public void testListIndexesWithUnreachableIndex_KATTA_76() throws Exception { 38 | IndexMetaData indexMD = new IndexMetaData("indexABC", "hdfs://localhost:8020/unreachableIndex", 1); 39 | _zk.getInteractionProtocol().publishIndex(indexMD); 40 | execute(Katta.LIST_INDICES_COMMAND, indexMD.getName()); 41 | } 42 | 43 | private void execute(Command command, String... args) throws Exception { 44 | command.parseArguments(_zk.getZkConf(), args); 45 | command.execute(_zk.getZkConf()); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/client/DeployClientTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.client; 17 | 18 | import static org.mockito.Matchers.any; 19 | import static org.mockito.Mockito.mock; 20 | import static org.mockito.Mockito.verify; 21 | import net.sf.katta.AbstractTest; 22 | import net.sf.katta.operation.master.AbstractIndexOperation; 23 | import net.sf.katta.operation.master.IndexDeployOperation; 24 | import net.sf.katta.protocol.InteractionProtocol; 25 | 26 | import org.junit.Test; 27 | 28 | public class DeployClientTest extends AbstractTest { 29 | 30 | protected InteractionProtocol _protocol = mock(InteractionProtocol.class); 31 | 32 | @Test 33 | public void testAddIndex() throws Exception { 34 | DeployClient deployClient = new DeployClient(_protocol); 35 | deployClient.addIndex("i1", "iP", 1); 36 | verify(_protocol).addMasterOperation(any(IndexDeployOperation.class)); 37 | } 38 | 39 | @Test(expected = IllegalArgumentException.class) 40 | public void testAddIndexWithInvalidReplicationCount() throws Exception { 41 | DeployClient deployClient = new DeployClient(_protocol); 42 | deployClient.addIndex("i1", "iP", 0); 43 | } 44 | 45 | @Test(expected = IllegalArgumentException.class) 46 | public void testAddIndexWithInvalidName() throws Exception { 47 | DeployClient deployClient = new DeployClient(_protocol); 48 | deployClient.addIndex("*", "iP", 1); 49 | } 50 | 51 | @Test(expected = IllegalArgumentException.class) 52 | public void testAddIndexWithInvalidName2() throws Exception { 53 | DeployClient deployClient = new DeployClient(_protocol); 54 | deployClient.addIndex("a" + AbstractIndexOperation.INDEX_SHARD_NAME_SEPARATOR, "iP", 1); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/lib/lucene/FilterWritableTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.lucene; 17 | 18 | import net.sf.katta.AbstractWritableTest; 19 | 20 | import org.apache.hadoop.io.DataOutputBuffer; 21 | import org.apache.lucene.index.Term; 22 | import org.apache.lucene.search.QueryWrapperFilter; 23 | import org.apache.lucene.search.TermQuery; 24 | import org.junit.Test; 25 | 26 | import static org.junit.Assert.assertTrue; 27 | 28 | public class FilterWritableTest extends AbstractWritableTest { 29 | 30 | @Test 31 | public void testSerializeFilter() throws Exception { 32 | TermQuery termQuery = new TermQuery(new Term("katta")); 33 | QueryWrapperFilter filter = new QueryWrapperFilter(termQuery); 34 | FilterWritable writable = new FilterWritable(filter); 35 | DataOutputBuffer buffer = writeWritable(writable); 36 | 37 | FilterWritable writable2 = new FilterWritable(); 38 | readWritable(buffer, writable2); 39 | 40 | assertTrue(writable.equals(writable2)); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/lib/lucene/HitTest.java: -------------------------------------------------------------------------------- 1 | package net.sf.katta.lib.lucene; 2 | 3 | import java.util.List; 4 | 5 | import org.junit.Test; 6 | 7 | import com.google.common.collect.ImmutableList; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | 11 | public class HitTest { 12 | public static class HitPair { 13 | public final Hit hitA; 14 | public final Hit hitB; 15 | public final int compareValue; 16 | 17 | public HitPair(Hit hitA, Hit hitB, int compareValue) { 18 | this.hitA = hitA; 19 | this.hitB = hitB; 20 | this.compareValue = compareValue; 21 | } 22 | } 23 | 24 | public static List HIT_PAIRS = ImmutableList.of( 25 | new HitPair(new Hit("shard1", "node1", 1.0f, 1), new Hit("shard1", "node1", 0.8f, 1), -1), 26 | new HitPair(new Hit("shard1", "node1", 0.8f, 1), new Hit("shard1", "node1", 1.0f, 1), 1), 27 | new HitPair(new Hit("shard1", "node1", 1.0f, 1), new Hit("shard1", "node1", 1.0f, 2), -1), 28 | new HitPair(new Hit("shard1", "node1", 1.0f, 2), new Hit("shard1", "node1", 1.0f, 1), 1), 29 | new HitPair(new Hit("shard2", "node1", 1.0f, 1), new Hit("shard1", "node1", 1.0f, 1), -1), 30 | new HitPair(new Hit("shard1", "node1", 1.0f, 1), new Hit("shard1", "node1", 1.0f, 1), 0), 31 | new HitPair(new Hit("shard1", "node1", 1.0f, 1), new Hit("shard2", "node1", 1.0f, 1), 1) 32 | ); 33 | 34 | @Test 35 | public void testCompareTo() { 36 | for (HitPair hitPair: HIT_PAIRS) { 37 | assertEquals(String.format("%s.compareTo(%s): %d", hitPair.hitA, hitPair.hitB, hitPair.compareValue), 38 | hitPair.compareValue, hitPair.hitA.compareTo(hitPair.hitB)); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/lib/lucene/HitsMapWritableTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.lucene; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | 20 | import java.util.List; 21 | 22 | import net.sf.katta.AbstractWritableTest; 23 | 24 | import org.apache.hadoop.io.DataOutputBuffer; 25 | import org.junit.Test; 26 | 27 | public class HitsMapWritableTest extends AbstractWritableTest { 28 | 29 | @Test 30 | public void testSerialization() throws Exception { 31 | String nodeName = "node1"; 32 | HitsMapWritable hitsMapWritable = new HitsMapWritable(nodeName); 33 | String shard1 = "shard1"; 34 | String shard2 = "shard2"; 35 | String shard3 = "shard3"; 36 | for (int i = 0; i < 200; i++) { 37 | hitsMapWritable.addHit(new Hit(shard1, nodeName, i, 10000 + i)); 38 | hitsMapWritable.addHit(new Hit(shard2, nodeName, i, 20000 + i)); 39 | hitsMapWritable.addHit(new Hit(shard3, nodeName, i, 30000 + i)); 40 | } 41 | 42 | DataOutputBuffer out = writeWritable(hitsMapWritable); 43 | HitsMapWritable readWritable = (HitsMapWritable) readWritable(out, new HitsMapWritable()); 44 | 45 | assertEquals(hitsMapWritable.getNodeName(), readWritable.getNodeName()); 46 | assertEquals(hitsMapWritable.getTotalHits(), readWritable.getTotalHits()); 47 | List hits = hitsMapWritable.getHitList(); 48 | List readHits = readWritable.getHitList(); 49 | assertEquals(hits.size(), readHits.size()); 50 | for (int i = 0; i < hits.size(); i++) { 51 | assertEquals(hits.get(i), readHits.get(i)); 52 | } 53 | assertEquals(hitsMapWritable.getNodeName(), readWritable.getNodeName()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/lib/lucene/QueryWritableTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.lib.lucene; 17 | 18 | import static org.junit.Assert.assertTrue; 19 | import net.sf.katta.AbstractWritableTest; 20 | 21 | import org.apache.hadoop.io.DataOutputBuffer; 22 | import org.apache.lucene.index.Term; 23 | import org.apache.lucene.search.TermQuery; 24 | import org.junit.Test; 25 | 26 | public class QueryWritableTest extends AbstractWritableTest { 27 | 28 | @Test 29 | public void testSerializeQuery() throws Exception { 30 | TermQuery termQuery = new TermQuery(new Term("katta")); 31 | QueryWritable writable = new QueryWritable(termQuery); 32 | DataOutputBuffer buffer = writeWritable(writable); 33 | 34 | QueryWritable writable2 = new QueryWritable(); 35 | readWritable(buffer, writable2); 36 | 37 | assertTrue(writable.equals(writable2)); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/node/monitor/JmxMonitorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.node.monitor; 17 | 18 | import static org.mockito.Matchers.anyObject; 19 | import static org.mockito.Matchers.eq; 20 | import static org.mockito.Mockito.atLeastOnce; 21 | import static org.mockito.Mockito.mock; 22 | import static org.mockito.Mockito.verify; 23 | import net.sf.katta.AbstractTest; 24 | import net.sf.katta.node.monitor.JmxMonitor; 25 | import net.sf.katta.node.monitor.MetricsRecord; 26 | import net.sf.katta.protocol.InteractionProtocol; 27 | 28 | import org.junit.Test; 29 | 30 | public class JmxMonitorTest extends AbstractTest { 31 | 32 | @Test 33 | public void testGetCpu() throws Exception { 34 | JmxMonitor monitor = new JmxMonitor(); 35 | String nodeId = "someId"; 36 | InteractionProtocol protocol = mock(InteractionProtocol.class); 37 | monitor.startMonitoring(nodeId, protocol); 38 | Thread.sleep(1200); 39 | verify(protocol, atLeastOnce()).setMetric(eq(nodeId), (MetricsRecord) anyObject()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/node/monitor/MetricLoggerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.node.monitor; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | import net.sf.katta.AbstractZkTest; 20 | import net.sf.katta.node.monitor.MetricLogger.OutputType; 21 | import net.sf.katta.protocol.InteractionProtocol; 22 | 23 | import org.junit.Test; 24 | 25 | public class MetricLoggerTest extends AbstractZkTest { 26 | 27 | @Test 28 | public void testLogMetric() throws Exception { 29 | InteractionProtocol protocol = _zk.createInteractionProtocol(); 30 | MetricLogger metricLogger = new MetricLogger(OutputType.Log4J, protocol); 31 | protocol.setMetric("node1", new MetricsRecord("node1")); 32 | Thread.sleep(500); 33 | protocol.setMetric("node1", new MetricsRecord("node1")); 34 | Thread.sleep(500); 35 | assertEquals(2, metricLogger.getLoggedRecords()); 36 | protocol.unregisterComponent(metricLogger); 37 | protocol.disconnect(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/operation/node/AbstractNodeOperationMockTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.operation.node; 17 | 18 | import static org.mockito.Mockito.mock; 19 | import net.sf.katta.AbstractTest; 20 | import net.sf.katta.node.IContentServer; 21 | import net.sf.katta.node.Node; 22 | import net.sf.katta.node.NodeContext; 23 | import net.sf.katta.node.ShardManager; 24 | import net.sf.katta.protocol.InteractionProtocol; 25 | import net.sf.katta.testutil.Mocks; 26 | 27 | public class AbstractNodeOperationMockTest extends AbstractTest { 28 | 29 | protected InteractionProtocol _protocol = mock(InteractionProtocol.class); 30 | protected Node _node = Mocks.mockNode(); 31 | protected ShardManager _shardManager = mock(ShardManager.class); 32 | protected IContentServer _contentServer = mock(IContentServer.class); 33 | protected NodeContext _context = new NodeContext(_protocol, _node, _shardManager, _contentServer); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/PrintMethodNames.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil; 17 | 18 | import org.apache.log4j.Logger; 19 | import org.junit.rules.MethodRule; 20 | import org.junit.runners.model.FrameworkMethod; 21 | import org.junit.runners.model.Statement; 22 | 23 | public class PrintMethodNames implements MethodRule { 24 | 25 | protected static final Logger LOG = Logger.getLogger(PrintMethodNames.class); 26 | 27 | private String _methodName; 28 | 29 | @Override 30 | public Statement apply(final Statement statement, final FrameworkMethod method, Object target) { 31 | _methodName = method.getName(); 32 | return new Statement() { 33 | @Override 34 | public void evaluate() throws Throwable { 35 | String testClassName = method.getMethod().getDeclaringClass().getName(); 36 | LOG.info("~~~~~~~~~~~~~~~ " + testClassName + "#" + method.getName() + "() [" + Thread.activeCount() + "] ~"); 37 | statement.evaluate(); 38 | LOG 39 | .info("~~~~~~~~~~~~~~~ FIN " + testClassName + "#" + method.getName() + "() [" + Thread.activeCount() 40 | + "]~"); 41 | } 42 | }; 43 | } 44 | 45 | public String getCurrentMethodName() { 46 | return _methodName; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/TestIoUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil; 17 | 18 | import java.io.BufferedReader; 19 | import java.io.BufferedWriter; 20 | import java.io.File; 21 | import java.io.FileReader; 22 | import java.io.FileWriter; 23 | import java.io.IOException; 24 | 25 | public class TestIoUtil { 26 | 27 | public static void writeFile(File file, String... lines) throws IOException { 28 | BufferedWriter writer = new BufferedWriter(new FileWriter(file)); 29 | for (String line : lines) { 30 | writer.append(line); 31 | writer.newLine(); 32 | } 33 | writer.close(); 34 | } 35 | 36 | public static long countLines(File file) throws IOException { 37 | BufferedReader reader = new BufferedReader(new FileReader(file)); 38 | long lines = 0; 39 | while (reader.readLine() != null) { 40 | lines++; 41 | } 42 | reader.close(); 43 | return lines; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/TestResources.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil; 17 | 18 | import java.io.File; 19 | 20 | public class TestResources { 21 | 22 | public final static File INDEX1 = new File("./src/test/testIndexA"); 23 | public final static File INDEX2 = new File("./src/test/testIndexB"); 24 | 25 | public final static File UNZIPPED_INDEX = new File("./src/test/testIndexC"); 26 | public final static File INVALID_INDEX = new File("./src/test/testIndexInvalid"); 27 | 28 | public final static File SHARD1 = new File(INDEX2, "aIndex.zip"); 29 | 30 | public final static File MAP_FILE_A = new File("./src/test/testMapFileA"); 31 | public final static File MAP_FILE_B = new File("./src/test/testMapFileB"); 32 | 33 | /** The shards will be created at run time. */ 34 | public final static File EMPTY1_INDEX = new File("./build/data/TestResources/empty1"); 35 | /** The shards will be created at run time. */ 36 | public final static File EMPTY2_INDEX = new File("./build/data/TestResources/empty2"); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/ChainedAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import org.mockito.invocation.InvocationOnMock; 19 | import org.mockito.stubbing.Answer; 20 | 21 | /** 22 | * Executes a given list of {@link Answer}s. Returns the return-value of the 23 | * last answer. 24 | */ 25 | @SuppressWarnings("unchecked") 26 | public class ChainedAnswer implements Answer { 27 | 28 | private final Answer[] _answers; 29 | 30 | public ChainedAnswer(Answer... answers) { 31 | _answers = answers; 32 | } 33 | 34 | @Override 35 | public Object answer(InvocationOnMock invocation) throws Throwable { 36 | Object returnValue = null; 37 | for (Answer answer : _answers) { 38 | returnValue = answer.answer(invocation); 39 | } 40 | return returnValue; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/PauseAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import org.mockito.invocation.InvocationOnMock; 19 | import org.mockito.stubbing.Answer; 20 | 21 | public class PauseAnswer implements Answer { 22 | 23 | private volatile boolean _executionStarted; 24 | private volatile boolean _pauseCanceled; 25 | private final T _returnValue; 26 | private Exception _resumeException; 27 | 28 | public PauseAnswer(T returnValue) { 29 | _returnValue = returnValue; 30 | } 31 | 32 | @Override 33 | public synchronized T answer(InvocationOnMock invocation) throws Throwable { 34 | notifyAll();// for joinExecutionBegin 35 | _executionStarted = true; 36 | if (!_pauseCanceled) { 37 | wait(); 38 | } 39 | if (_resumeException != null) { 40 | throw _resumeException; 41 | } 42 | return _returnValue; 43 | } 44 | 45 | public synchronized void joinExecutionBegin() { 46 | try { 47 | if (!_executionStarted) { 48 | wait(); 49 | } 50 | } catch (InterruptedException e) { 51 | e.printStackTrace(); 52 | } 53 | } 54 | 55 | public synchronized void resumeExecution(boolean cancelAllFurtherPauses) { 56 | notifyAll(); 57 | _executionStarted = false; 58 | if (cancelAllFurtherPauses) { 59 | _pauseCanceled = true; 60 | } 61 | } 62 | 63 | public void resumeExecutionWithException(boolean cancelAllFurtherPauses, Exception resumeException) { 64 | _resumeException = resumeException; 65 | resumeExecution(cancelAllFurtherPauses); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/PrintStacktraceAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import java.util.Arrays; 19 | 20 | import org.mockito.invocation.InvocationOnMock; 21 | import org.mockito.stubbing.Answer; 22 | 23 | /** 24 | * Useful to find out who called a given mock method.
25 | * Usage:
26 | * doAnswer(new PrintStacktraceAnswer()).when(mock).someMethod(notNull()); 27 | * 28 | */ 29 | public class PrintStacktraceAnswer implements Answer { 30 | 31 | @Override 32 | public Void answer(InvocationOnMock invocation) throws Throwable { 33 | System.out.println("called " + invocation.getMethod() + " with arguments " 34 | + Arrays.asList(invocation.getArguments())); 35 | StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); 36 | for (StackTraceElement string : stackTrace) { 37 | System.out.println("\t" + string.toString()); 38 | } 39 | return null; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/SerializableCountDownLatchAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import java.io.Serializable; 19 | import java.util.concurrent.CountDownLatch; 20 | 21 | import org.mockito.invocation.InvocationOnMock; 22 | import org.mockito.stubbing.Answer; 23 | 24 | @SuppressWarnings("serial") 25 | public class SerializableCountDownLatchAnswer implements Answer, Serializable { 26 | 27 | private static CountDownLatch _countDownLatch; 28 | 29 | public SerializableCountDownLatchAnswer(int latchCount) { 30 | _countDownLatch = new CountDownLatch(latchCount); 31 | } 32 | 33 | @Override 34 | public Void answer(InvocationOnMock invocation) throws Throwable { 35 | _countDownLatch.countDown(); 36 | return null; 37 | } 38 | 39 | public CountDownLatch getCountDownLatch() { 40 | return _countDownLatch; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/SerializableThrowsExceptionAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import java.io.Serializable; 19 | 20 | import org.mockito.invocation.InvocationOnMock; 21 | import org.mockito.stubbing.Answer; 22 | 23 | public class SerializableThrowsExceptionAnswer implements Answer, Serializable { 24 | 25 | private static final long serialVersionUID = 1L; 26 | private final Exception _exception; 27 | 28 | public SerializableThrowsExceptionAnswer(Exception exception) { 29 | _exception = exception; 30 | } 31 | 32 | @Override 33 | public T answer(InvocationOnMock invocation) throws Throwable { 34 | throw _exception; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/SleepingAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import org.mockito.invocation.InvocationOnMock; 19 | import org.mockito.stubbing.Answer; 20 | 21 | public class SleepingAnswer implements Answer { 22 | 23 | private final long _timeToSleep; 24 | 25 | public SleepingAnswer() { 26 | this(Long.MAX_VALUE); 27 | } 28 | 29 | public SleepingAnswer(long timeToSleep) { 30 | _timeToSleep = timeToSleep; 31 | } 32 | 33 | @Override 34 | public Void answer(InvocationOnMock invocation) throws Throwable { 35 | Thread.sleep(_timeToSleep); 36 | return null; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/testutil/mockito/WaitingAnswer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.testutil.mockito; 17 | 18 | import org.mockito.invocation.InvocationOnMock; 19 | import org.mockito.stubbing.Answer; 20 | 21 | public class WaitingAnswer implements Answer { 22 | 23 | @Override 24 | public Void answer(InvocationOnMock invocation) throws Throwable { 25 | synchronized (this) { 26 | this.wait(); 27 | } 28 | return null; 29 | } 30 | 31 | public void release() { 32 | synchronized (this) { 33 | this.notifyAll(); 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/tool/SampleIndexGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package net.sf.katta.tool; 18 | 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.io.File; 22 | 23 | import net.sf.katta.AbstractTest; 24 | import net.sf.katta.testutil.TestIoUtil; 25 | 26 | import org.apache.lucene.index.IndexReader; 27 | import org.apache.lucene.store.FSDirectory; 28 | import org.junit.Test; 29 | 30 | public class SampleIndexGeneratorTest extends AbstractTest { 31 | 32 | @Test 33 | public void testCreateIndex() throws Exception { 34 | SampleIndexGenerator sampleIndexGenerator = new SampleIndexGenerator(); 35 | File inputFile = _temporaryFolder.newFile("inputFile"); 36 | TestIoUtil.writeFile(inputFile, "Project Gutenberg's Alice's Adventures in Wonderland, by Lewis Carroll", 37 | "Title: Alice's Adventures in Wonderland"); 38 | File file = _temporaryFolder.newFolder("sampleIndex"); 39 | file.mkdirs(); 40 | sampleIndexGenerator.createIndex(inputFile.getAbsolutePath(), file.getAbsolutePath(), 10, 10); 41 | assertTrue(IndexReader.indexExists(FSDirectory.open(file.listFiles()[0]))); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/tool/loadtest/LoadTestNodeOperationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.tool.loadtest; 17 | 18 | import static org.hamcrest.Matchers.isIn; 19 | import static org.junit.Assert.assertThat; 20 | import static org.mockito.Matchers.argThat; 21 | import static org.mockito.Matchers.eq; 22 | import static org.mockito.Mockito.mock; 23 | import static org.mockito.Mockito.verify; 24 | import static org.mockito.Mockito.when; 25 | import net.sf.katta.operation.node.AbstractNodeOperationMockTest; 26 | import net.sf.katta.testutil.mockito.AlmostVerificationMode; 27 | import net.sf.katta.tool.loadtest.query.AbstractQueryExecutor; 28 | 29 | import org.junit.Test; 30 | 31 | public class LoadTestNodeOperationTest extends AbstractNodeOperationMockTest { 32 | 33 | @Test 34 | public void testNodeOperation() throws Exception { 35 | AbstractQueryExecutor queryExecutor = mock(AbstractQueryExecutor.class); 36 | String[] queries = new String[] { "a", "b", "c" }; 37 | when(queryExecutor.getQueries()).thenReturn(queries); 38 | final int queryRate = 10; 39 | final int runTime = 2000; 40 | LoadTestNodeOperation nodeOperation = new LoadTestNodeOperation(queryExecutor, queryRate, runTime); 41 | long startTime = System.currentTimeMillis(); 42 | nodeOperation.execute(_context); 43 | long endTime = System.currentTimeMillis(); 44 | assertThat(endTime - startTime, almostEquals(runTime, 500)); 45 | 46 | verify(queryExecutor).init(_context); 47 | verify(queryExecutor, new AlmostVerificationMode(queryRate * runTime / 1000, 5)).execute(eq(_context), 48 | argThat(isIn(queries))); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/util/ClassUtilTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import org.junit.Test; 19 | 20 | import static org.junit.Assert.fail; 21 | 22 | import static org.fest.assertions.Assertions.assertThat; 23 | 24 | public class ClassUtilTest { 25 | 26 | @Test 27 | public void testPrivateField() throws Exception { 28 | TestClass1 testClass1 = new TestClass1(); 29 | TestClass2 testClass2 = new TestClass2(); 30 | 31 | assertThat(ClassUtil.getPrivateFieldValue(testClass1, "field1")).isEqualTo("1"); 32 | assertThat(ClassUtil.getPrivateFieldValue(testClass2, "field2")).isEqualTo("2"); 33 | assertThat(ClassUtil.getPrivateFieldValue(testClass2, "field1")).isEqualTo("1"); 34 | 35 | try { 36 | ClassUtil.getPrivateFieldValue(testClass2, "fieldXY"); 37 | fail("should throw exception"); 38 | } catch (Exception e) { 39 | // expected 40 | } 41 | } 42 | 43 | private static class TestClass1 { 44 | @SuppressWarnings("unused") 45 | private String field1 = "1"; 46 | } 47 | 48 | private static class TestClass2 extends TestClass1 { 49 | @SuppressWarnings("unused") 50 | private String field2 = "2"; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/util/ConcurrentOne2ManyListMapTest.java: -------------------------------------------------------------------------------- 1 | package net.sf.katta.util; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertFalse; 5 | import static org.junit.Assert.assertTrue; 6 | 7 | import java.util.Iterator; 8 | import java.util.Set; 9 | 10 | import org.junit.Test; 11 | 12 | public class ConcurrentOne2ManyListMapTest { 13 | @Test 14 | public void testMap() throws Exception { 15 | ConcurrentOne2ManyListMap multiMap = new ConcurrentOne2ManyListMap(); 16 | assertEquals(0, multiMap.size()); 17 | assertEquals(0, multiMap.asMap().size()); 18 | assertTrue(multiMap.getValues("notExistentKey").isEmpty()); 19 | 20 | multiMap.add("a", 1); 21 | assertEquals(1, multiMap.size()); 22 | assertEquals(1, multiMap.asMap().size()); 23 | assertFalse(multiMap.getValues("a").isEmpty()); 24 | assertEquals(new Integer(1), multiMap.getValues("a").get(0)); 25 | } 26 | 27 | @Test 28 | public void testConcurrentAdd() throws InterruptedException { 29 | final ConcurrentOne2ManyListMap multiMap = new ConcurrentOne2ManyListMap(); 30 | 31 | multiMap.add(1); 32 | Set keys = multiMap.keySet(); 33 | Iterator iterator = keys.iterator(); 34 | multiMap.add(2); 35 | iterator.next(); 36 | } 37 | 38 | @Test 39 | public void testConcurrentAddEntry() throws InterruptedException { 40 | final ConcurrentOne2ManyListMap multiMap = new ConcurrentOne2ManyListMap(); 41 | 42 | for (int i = 0; i < 10; i++) { 43 | multiMap.add(1, i); 44 | } 45 | 46 | Iterator iterator = multiMap.getValues(1).iterator(); 47 | 48 | for (int i = 0; i < 10; i++) { 49 | multiMap.add(1, i); 50 | } 51 | 52 | iterator.next(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/util/ISleepServer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import org.apache.hadoop.ipc.VersionedProtocol; 19 | 20 | /** 21 | * The public interface for the back end of a dummy server that just 22 | * sleeps for a while then returns null. Used for testing. 23 | */ 24 | public interface ISleepServer extends VersionedProtocol { 25 | 26 | /** 27 | * Sleep for the given number of milliseconds, +- a random delta. 28 | * @param msec How long each node should sleep for. 29 | * @param delta The maximum size of the delta (msec) to add or remove 30 | * from the specified time. The node will choose an evenly distributed 31 | * random sleep time from msec-delta to msec+delta. 32 | * @param shards Which shards to use. This is only for testing. The sleep 33 | * happens on a per-node basis. If invalid shards are passed in, an 34 | * IllegalArgumentException is thrown. Otherwise this parameter is ignored. 35 | * No checking is done if the value is null. 36 | * @return the number of shards used (note: sleeping is only done once). 37 | * @throws IllegalArgumentException if invalid shard names are passed in. 38 | */ 39 | public int sleep(long msec, int delta, String[] shards) throws IllegalArgumentException; 40 | 41 | } 42 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/util/One2ManyListMapTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | import static org.junit.Assert.assertFalse; 20 | import static org.junit.Assert.assertTrue; 21 | import net.sf.katta.AbstractTest; 22 | 23 | import org.junit.Test; 24 | 25 | public class One2ManyListMapTest extends AbstractTest { 26 | 27 | @Test 28 | public void testMap() throws Exception { 29 | One2ManyListMap multiMap = new One2ManyListMap(); 30 | assertEquals(0, multiMap.size()); 31 | assertEquals(0, multiMap.asMap().size()); 32 | assertTrue(multiMap.getValues("notExistentKey").isEmpty()); 33 | 34 | multiMap.add("a", 1); 35 | assertEquals(1, multiMap.size()); 36 | assertEquals(1, multiMap.asMap().size()); 37 | assertFalse(multiMap.getValues("a").isEmpty()); 38 | assertEquals(new Integer(1), multiMap.getValues("a").get(0)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/java/net/sf/katta/util/ZkConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | import static org.junit.Assert.fail; 20 | import net.sf.katta.AbstractTest; 21 | 22 | import org.junit.Test; 23 | 24 | public class ZkConfigurationTest extends AbstractTest { 25 | 26 | @Test 27 | public void testSystemProperty() { 28 | try { 29 | System.clearProperty(ZkConfiguration.KATTA_PROPERTY_NAME); 30 | ZkConfiguration conf1 = new ZkConfiguration(); 31 | System.setProperty(ZkConfiguration.KATTA_PROPERTY_NAME, "/katta.zk.properties_alt_root"); 32 | ZkConfiguration conf2 = new ZkConfiguration(); 33 | // 34 | assertEquals("/katta", conf1.getZkRootPath()); 35 | assertEquals("/test/katta20090510153800", conf2.getZkRootPath()); 36 | // 37 | try { 38 | System.setProperty(ZkConfiguration.KATTA_PROPERTY_NAME, "/not-found"); 39 | new ZkConfiguration(); 40 | fail("Should have failed"); 41 | } catch (RuntimeException e) { 42 | // Good. 43 | } 44 | } finally { 45 | System.clearProperty(ZkConfiguration.KATTA_PROPERTY_NAME); 46 | } 47 | } 48 | 49 | @Test 50 | public void testZkParent() { 51 | assertEquals("/katta", ZkConfiguration.getZkParent("/katta/abc")); 52 | assertEquals("/", ZkConfiguration.getZkParent("/katta")); 53 | assertEquals(null, ZkConfiguration.getZkParent("/")); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/katta.master.properties: -------------------------------------------------------------------------------- 1 | # used distribution policy 2 | master.deploy.policy=net.sf.katta.master.DefaultDistributionPolicy 3 | safemode.maxTime=500 -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/katta.node.properties: -------------------------------------------------------------------------------- 1 | # the start port to try 2 | node.server.port.start = 20000 3 | # local folder on node where shards will be stored during serving 4 | node.shard.folder=/tmp/katta-shards 5 | node.monitor.class=net.sf.katta.node.monitor.JmxMonitor 6 | node.server.class=net.sf.katta.lib.lucene.LuceneServer 7 | 8 | # uncomment to enable throttling of bandwith in kilo-bytes/sec for shard deployments. In kbytes/sec. 9 | #node.shard.deploy.throttle= 10 | 11 | 12 | ### ~~~~~~~~~~~~~~~~~~~~~~~ ### 13 | ### Lucene related settings ### 14 | ### ~~~~~~~~~~~~~~~~~~~~~~~ ### 15 | 16 | # the percentage (value between 0 and 1) of the client-timeout which should be used 17 | # as server-timeout (as restriction time for the hit-collector). 18 | # Disable the timeout setting the value to 0. 19 | lucene.collector.timeout-percentage=0.75 20 | 21 | lucene.searcher.factory-class=net.sf.katta.lib.lucene.DefaultSearcherFactory 22 | lucene.searcher.threadpool.core-size=25 23 | lucene.searcher.threadpool.max-size=100 -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/katta.zk.properties: -------------------------------------------------------------------------------- 1 | # Starts zookeeper embedded in the master jvm. 2 | # We suggest that you run zookeeper standalone in large installations. 3 | # See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperAdmin.html 4 | zookeeper.embedded=true 5 | 6 | # Comma separated list of host:port of zookeeper servers used by the zookeeper clients. 7 | # Make sure the port is accessible from all nodes and use hostnames and not ip addresses. 8 | # If embedded is set to true, the server hostname is compared to the servers list. 9 | # If found an embedded zookeeper server is started within the master or secondary master jvm. 10 | zookeeper.servers=localhost:2181 11 | 12 | 13 | # The root zk node. This changes with every new cluster. 14 | zookeeper.root-path=/katta 15 | #zookeeper client timeout in milliseconds 16 | zookeeper.timeout=1000 17 | #zookeeper tick time 18 | zookeeper.tick-time=2000 19 | # zookeeper init time limit 20 | zookeeper.init-limit=5 21 | # zookeeper sync limit 22 | zookeeper.sync-limit=2 23 | # zookeeper folder where data are stored 24 | zookeeper.data-dir=./zookeeper-data 25 | # zookeeper folder where log data are stored 26 | zookeeper.log-data-dir=./zookeeper-log-data 27 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/katta.zk.properties_alt_root: -------------------------------------------------------------------------------- 1 | # comma serperated list of host:port that should run a zookeeper server, make sure you use hostnames and not ip addresses 2 | zookeeper.servers=localhost:2181 3 | #zookeeper client timeout in milliseconds 4 | zookeeper.timeout=5000 5 | #zookeeper tick time 6 | zookeeper.tick-time=2000 7 | # zookeeper init time limit 8 | zookeeper.init-limit=5 9 | # zookeeper sync limit 10 | zookeeper.sync-limit=2 11 | # zookeeper folder where data are stored 12 | zookeeper.data-dir=build/zookeeper-data 13 | # zookeeper folder where log data are stored 14 | zookeeper.log-data-dir=build/zookeeper-log-data 15 | # Katta root node to use 16 | zookeeper.root-path=/test/katta20090510153800 17 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | katta.root.logger=INFO,console, file 3 | log4j.logger.org.apache.zookeeper=ERROR 4 | log4j.logger.org.apache.hadoop=WARN 5 | log4j.logger.org.mortbay=WARN 6 | org.I0Itec.zkclient=INFO 7 | katta.log.dir=. 8 | katta.log.file=katta.log 9 | 10 | 11 | # Define the root logger to the system property "katta.root.logger". 12 | log4j.rootLogger=${katta.root.logger} 13 | 14 | # Logging Threshold 15 | log4j.threshhold=ALL 16 | 17 | # 18 | # Daily Rolling File Appender 19 | # 20 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 21 | log4j.appender.DRFA.File=${katta.log.dir}/${katta.log.file} 22 | 23 | # Rollver at midnight 24 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 25 | 26 | # 30-day backup 27 | #log4j.appender.DRFA.MaxBackupIndex=30 28 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 29 | 30 | # Pattern format: Date LogLevel LoggerName LogMessage 31 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c:%L - %m%n 32 | 33 | # Debugging Pattern format 34 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 35 | 36 | 37 | # 38 | # console 39 | # Add "console" to rootlogger above if you want to use this 40 | # 41 | log4j.appender.console=org.apache.log4j.ConsoleAppender 42 | log4j.appender.console.target=System.out 43 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 44 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss,SSS} %p %c{2}:%L - %m%n 45 | 46 | #file appender 47 | log4j.appender.file=org.apache.log4j.FileAppender 48 | log4j.appender.file.File=log.log 49 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 50 | log4j.appender.file.layout.ConversionPattern=%5p [%d{yyyy-MM-dd HH:mm:ss}] (%F:%L) - %m%n 51 | 52 | # Custom Logging levels 53 | 54 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 55 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/user.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | foo 4 | bar 5 | 6 | 29 7 | foobar 8 | pwd 9 | 10 | -------------------------------------------------------------------------------- /modules/katta-core/src/test/resources/xpath.properties: -------------------------------------------------------------------------------- 1 | /user/name/first=STRING 2 | /user/age=NUMBER -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/aIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/aIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/aIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/aIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/aIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/aIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/bIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/bIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/bIndex/copy.done: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/bIndex/copy.done -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/bIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/bIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/bIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/bIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/cIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/cIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/cIndex/copy.done: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/cIndex/copy.done -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/cIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/cIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/cIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/cIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/dIndex/_us.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/dIndex/_us.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexA/dIndex/segments: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexA/dIndex/segments -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexB/aIndex.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexB/aIndex.zip -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexB/bIndex.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexB/bIndex.zip -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexB/cIndex.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexB/cIndex.zip -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexB/dindex.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexB/dindex.zip -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexC/aIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexC/aIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexC/aIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexC/aIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexC/aIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexC/aIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/aIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/aIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/bIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/bIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/bIndex/copy.done: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/bIndex/copy.done -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/bIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/bIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/bIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/bIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/cIndex/_0.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/cIndex/_0.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/cIndex/copy.done: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/cIndex/copy.done -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/cIndex/segments.gen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/cIndex/segments.gen -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/cIndex/segments_3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/cIndex/segments_3 -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/dIndex/_us.cfs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/dIndex/_us.cfs -------------------------------------------------------------------------------- /modules/katta-core/src/test/testIndexInvalid/dIndex/segments: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testIndexInvalid/dIndex/segments -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a1/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a1/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a1/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a1/index -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a2/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a2/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a2/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a2/index -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a3/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a3/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a3/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a3/index -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a4/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a4/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileA/a4/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileA/a4/index -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileB/b1/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileB/b1/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileB/b1/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileB/b1/index -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileB/b2/data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileB/b2/data -------------------------------------------------------------------------------- /modules/katta-core/src/test/testMapFileB/b2/index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sgroschupf/katta/db35d877e327d04c8b3a5fb5edb50d73f0cd207c/modules/katta-core/src/test/testMapFileB/b2/index -------------------------------------------------------------------------------- /modules/katta-ec2/README.txt: -------------------------------------------------------------------------------- 1 | Katta EC2 2 | 3 | This scripts are a port of the hadoop ec2 scripts. They are only minimal customized. 4 | Credits go to the hadoop ec2 script authors. 5 | 6 | 7 | This collection of scripts allows you to run Katta clusters on Amazon.com's Elastic Compute Cloud (EC2) service described at: 8 | 9 | http://aws.amazon.com/ec2 10 | 11 | To get help, type the following in a shell: 12 | 13 | bin/katta-ec2 14 | 15 | For full instructions, please visit the Hadoop wiki at: 16 | 17 | TODO sg 12/17/08 link to the right katta documentation 18 | 19 | http://wiki.apache.org/hadoop/AmazonEC2#AutomatedScripts 20 | -------------------------------------------------------------------------------- /modules/katta-ec2/bin/delete-katta-cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Delete the groups an local files associated with a cluster. 19 | 20 | if [ -z $1 ]; then 21 | echo "Cluster name required!" 22 | exit -1 23 | fi 24 | 25 | CLUSTER=$1 26 | 27 | # Finding Katta clusters 28 | CLUSTERS=`ec2-describe-instances | \ 29 | awk '"RESERVATION" == $1 && $4 ~ /-master$/, "INSTANCE" == $1' | tr '\n' '\t' | \ 30 | grep "$CLUSTER" | grep running | cut -f4 | rev | cut -d'-' -f2- | rev` 31 | 32 | if [ -n "$CLUSTERS" ]; then 33 | echo "Cluster $CLUSTER has running instances. Please terminate them first." 34 | exit 0 35 | fi 36 | 37 | # Import variables 38 | bin=`dirname "$0"` 39 | bin=`cd "$bin"; pwd` 40 | . "$bin"/katta-ec2-env.sh 41 | 42 | rm -f $MASTER_IP_PATH 43 | rm -f $MASTER_PRIVATE_IP_PATH 44 | 45 | ec2-describe-group | egrep "[[:space:]]$CLUSTER_MASTER[[:space:]]" > /dev/null 46 | if [ $? -eq 0 ]; then 47 | echo "Deleting group $CLUSTER_MASTER" 48 | ec2-revoke $CLUSTER_MASTER -o $CLUSTER -u $AWS_ACCOUNT_ID 49 | fi 50 | 51 | ec2-describe-group | egrep "[[:space:]]$CLUSTER[[:space:]]" > /dev/null 52 | if [ $? -eq 0 ]; then 53 | echo "Deleting group $CLUSTER" 54 | ec2-revoke $CLUSTER -o $CLUSTER_MASTER -u $AWS_ACCOUNT_ID 55 | fi 56 | 57 | ec2-delete-group $CLUSTER_MASTER 58 | ec2-delete-group $CLUSTER 59 | -------------------------------------------------------------------------------- /modules/katta-ec2/bin/image/ec2-run-user-data: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # ec2-run-user-data - Run instance user-data if it looks like a script. 4 | # 5 | # Only retrieves and runs the user-data script once per instance. If 6 | # you want the user-data script to run again (e.g., on the next boot) 7 | # then add this command in the user-data script: 8 | # rm -f /var/ec2/ec2-run-user-data.* 9 | # 10 | # History: 11 | # 2008-05-16 Eric Hammond 12 | # - Initial version including code from Kim Scheibel, Jorge Oliveira 13 | # 2008-08-06 Tom White 14 | # - Updated to use mktemp on fedora 15 | # 16 | 17 | prog=$(basename $0) 18 | logger="logger -t $prog" 19 | curl="curl --retry 3 --silent --show-error --fail" 20 | instance_data_url=http://169.254.169.254/2008-02-01 21 | 22 | # Wait until networking is up on the EC2 instance. 23 | perl -MIO::Socket::INET -e ' 24 | until(new IO::Socket::INET("169.254.169.254:80")){print"Waiting for network...\n";sleep 1} 25 | ' | $logger 26 | 27 | # Exit if we have already run on this instance (e.g., previous boot). 28 | ami_id=$($curl $instance_data_url/meta-data/ami-id) 29 | been_run_file=/var/ec2/$prog.$ami_id 30 | mkdir -p $(dirname $been_run_file) 31 | if [ -f $been_run_file ]; then 32 | $logger < $been_run_file 33 | exit 34 | fi 35 | 36 | # Retrieve the instance user-data and run it if it looks like a script 37 | user_data_file=`mktemp -t ec2-user-data.XXXXXXXXXX` 38 | chmod 700 $user_data_file 39 | $logger "Retrieving user-data" 40 | $curl -o $user_data_file $instance_data_url/user-data 2>&1 | $logger 41 | if [ ! -s $user_data_file ]; then 42 | $logger "No user-data available" 43 | elif head -1 $user_data_file | egrep -v '^#!'; then 44 | $logger "Skipping user-data as it does not begin with #!" 45 | else 46 | $logger "Running user-data" 47 | echo "user-data has already been run on this instance" > $been_run_file 48 | $user_data_file 2>&1 | logger -t "user-data" 49 | $logger "user-data exit code: $?" 50 | fi 51 | rm -f $user_data_file 52 | -------------------------------------------------------------------------------- /modules/katta-ec2/bin/launch-katta-cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Launch an EC2 cluster of Kata instances. 19 | 20 | # Import variables 21 | bin=`dirname "$0"` 22 | bin=`cd "$bin"; pwd` 23 | 24 | if [ -z $1 ]; then 25 | echo "Cluster name required!" 26 | exit -1 27 | fi 28 | 29 | if [ -z $2 ]; then 30 | echo "Must specify the number of nodes to start." 31 | exit -1 32 | fi 33 | 34 | if ! "$bin"/launch-katta-master $1 ; then 35 | exit $? 36 | fi 37 | 38 | if ! "$bin"/launch-katta-nodes $*; then 39 | exit $? 40 | fi 41 | -------------------------------------------------------------------------------- /modules/katta-ec2/bin/list-katta-clusters: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # List running clusters. 19 | 20 | # Import variables 21 | bin=`dirname "$0"` 22 | bin=`cd "$bin"; pwd` 23 | . "$bin"/katta-ec2-env.sh 24 | 25 | # Finding Katta clusters 26 | CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 && $4 ~ /-master$/, "INSTANCE" == $1' | tr '\n' '\t' | grep running | cut -f4 | rev | cut -d'-' -f2- | rev` 27 | 28 | [ -z "$CLUSTERS" ] && echo "No running clusters." && exit 0 29 | 30 | echo "Running Katta clusters:" 31 | echo "$CLUSTERS" 32 | -------------------------------------------------------------------------------- /modules/katta-ec2/bin/terminate-katta-cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Terminate a cluster. 19 | 20 | if [ -z $1 ]; then 21 | echo "Cluster name required!" 22 | exit -1 23 | fi 24 | 25 | CLUSTER=$1 26 | 27 | # Import variables 28 | bin=`dirname "$0"` 29 | bin=`cd "$bin"; pwd` 30 | . "$bin"/katta-ec2-env.sh 31 | 32 | # Finding Katta instance 33 | KATTA_INSTANCES=`ec2-describe-instances | awk '"RESERVATION" == $1 && ("'$CLUSTER'" == $4 || "'$CLUSTER_MASTER'" == $4), "RESERVATION" == $1 && ("'$CLUSTER'" != $4 && "'$CLUSTER_MASTER'" != $4)'` 34 | KATTA_INSTANCES=`echo "$KATTA_INSTANCES" | grep INSTANCE | grep running` 35 | 36 | [ -z "$KATTA_INSTANCES" ] && echo "No running instances in cluster $CLUSTER." && exit 0 37 | 38 | echo "Running Katta instances:" 39 | echo "$KATTA_INSTANCES" 40 | 41 | if [ -z $2 ]; then 42 | read -p "Terminate all instances? [yes or no]: " answer 43 | if [ "$answer" != "yes" ]; then 44 | echo "scripted stoped" 45 | exit 1 46 | fi 47 | else 48 | if [ "$2" != "yes" ]; then 49 | exit 1 50 | fi 51 | fi 52 | 53 | ec2-terminate-instances `echo "$KATTA_INSTANCES" | awk '{print $2}'` 54 | -------------------------------------------------------------------------------- /modules/katta-indexing-sample/build.properties: -------------------------------------------------------------------------------- 1 | root.dir=../.. 2 | job.main.class=net.sf.katta.indexing.IndexerJob -------------------------------------------------------------------------------- /modules/katta-indexing-sample/build.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /modules/katta-indexing-sample/run.sh: -------------------------------------------------------------------------------- 1 | # adding indexing sample jar 2 | for i in `find build/artifacts -name "*.jar"` 3 | do 4 | echo "adding to classpath $i ..." 5 | CLASSPATH=${CLASSPATH}:$i 6 | done 7 | 8 | #adding hadoop libs 9 | for i in `find ../katta-core/lib/compile/hadoop -name "*.jar"` 10 | do 11 | echo "adding to classpath $i ..." 12 | CLASSPATH=${CLASSPATH}:$i 13 | done 14 | 15 | $JAVA_HOME/bin/java -cp ${CLASSPATH} "$@" 16 | -------------------------------------------------------------------------------- /modules/katta-indexing-sample/src/test/java/net/sf/katta/indexing/IndexerJobTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.indexing; 17 | 18 | import java.io.File; 19 | 20 | import org.apache.hadoop.fs.FileUtil; 21 | import org.junit.Test; 22 | 23 | public class IndexerJobTest { 24 | 25 | @Test 26 | public void testStartJob() throws Exception { 27 | IndexerJob indexerJob = new IndexerJob(); 28 | File tmp = new File("./build/extras/indexing/tmp/IndexerJobTest"); 29 | tmp.mkdirs(); 30 | File in = new File(tmp, "in"); 31 | File out = new File(tmp, "out"); 32 | FileUtil.fullyDelete(out); 33 | String alicePath = SearchPathUtil.findPath("sample-data/texts/alice.txt", "../../sample-data/texts/alice.txt"); 34 | String sampleText = SequenceFileCreator.getSampleText(alicePath); 35 | SequenceFileCreator creator = new SequenceFileCreator(); 36 | creator.create(in.getAbsolutePath(), sampleText, 100000); 37 | indexerJob.startIndexer(in.getAbsolutePath(), out.getAbsolutePath(), 5); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /modules/katta-indexing-sample/src/test/java/net/sf/katta/indexing/SearchPathUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.indexing; 17 | 18 | import java.io.File; 19 | import java.util.Arrays; 20 | 21 | public class SearchPathUtil { 22 | 23 | public static String findPath(String... paths) { 24 | for (String path : paths) { 25 | if (new File(path).exists()) { 26 | return path; 27 | } 28 | } 29 | throw new IllegalArgumentException("File not found in " + Arrays.asList(paths)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /modules/katta-indexing-sample/src/test/java/net/sf/katta/indexing/SequnceFileCreatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.indexing; 17 | 18 | import static org.junit.Assert.assertTrue; 19 | 20 | import java.io.File; 21 | 22 | import junit.framework.Assert; 23 | 24 | import org.junit.Test; 25 | 26 | public class SequnceFileCreatorTest { 27 | 28 | @Test 29 | public void testCreateSequnceFile() throws Exception { 30 | 31 | SequenceFileCreator creator = new SequenceFileCreator(); 32 | String path = "./build/extras/indexing/tmp/SequenceFileCreator/sequenceFile"; 33 | org.apache.hadoop.fs.FileUtil.fullyDelete(new File(path).getParentFile()); 34 | Assert.assertFalse(new File(path).exists()); 35 | 36 | String textPath = SearchPathUtil.findPath("sample-data/texts/alice.txt", "../../sample-data/texts/alice.txt"); 37 | String sampleText = SequenceFileCreator.getSampleText(textPath); 38 | int num = 1000000; 39 | creator.create(path, sampleText, num); 40 | assertTrue(new File(path).exists()); 41 | 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/build/ant/build.properties: -------------------------------------------------------------------------------- 1 | name=katta 2 | version=0.7.dev 3 | year=2012 4 | 5 | artifact.basename=${ant.project.name}-${version} 6 | jar.name=${artifact.basename}.jar 7 | jar.sources.name=${artifact.basename}-sources.jar 8 | jar.test.name=${artifact.basename}-test.jar 9 | jar.it.name=${artifact.basename}-it.jar 10 | 11 | main.src.dir=src/main/java 12 | test.src.dir=src/test/java 13 | it.src.dir=src/it/java 14 | it.src.dir=src/it/java 15 | build.src.dir=src/build 16 | 17 | main.res.dir=src/main/resources 18 | test.res.dir=src/test/resources 19 | it.res.dir=src/it/resources 20 | 21 | conf.dir=conf 22 | lib.dir=lib 23 | 24 | build.dir=build 25 | build.dir.main-classes=${build.dir}/classes-main 26 | build.dir.test-classes=${build.dir}/classes-test 27 | build.dir.it-classes=${build.dir}/classes-it 28 | build.dir.test-reports=${build.dir}/unit-reports 29 | build.dir.it-reports=${build.dir}/it-reports 30 | build.javadoc=${build.dir}/java-doc 31 | 32 | build.dir.main-classes-eclipse=${build.dir}/classes-main-eclipse 33 | build.dir.test-classes-eclipse=${build.dir}/classes-test-eclipse 34 | build.dir.it-classes-eclipse=${build.dir}/classes-it-eclipse 35 | 36 | artifact.dir=${build.dir}/artifacts 37 | 38 | 39 | # All reports go into this directory 40 | reports.dir=${build.dir}/reports 41 | 42 | # Instrumented classes are deposited into this directory 43 | instrumented.dir=${build.dir}/instrumented 44 | 45 | # Coverage reports are deposited into these directories 46 | coverage.html.dir=${reports.dir}/cobertura-html 47 | checkstyle.html.dir=${reports.dir}/checkstyle-reports 48 | 49 | 50 | javadoc.link.java=http://java.sun.com/j2se/1.6.0/docs/api/ 51 | 52 | javac.debug=on 53 | javac.optimize=on 54 | javac.deprecation=off 55 | javac.version=1.6 56 | javac.args= 57 | javac.args.warnings=-Xlint:none 58 | build.encoding=ISO-8859-1 59 | 60 | test.it.forkmode=perTest -------------------------------------------------------------------------------- /src/build/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/build/checkstyle/java.header: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ -------------------------------------------------------------------------------- /src/build/eclipse/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | #Tue Jul 12 13:02:06 CEST 2011 2 | eclipse.preferences.version=1 3 | formatter_profile=Katta 4 | formatter_settings_version=11 5 | org.eclipse.jdt.ui.ignorelowercasenames=true 6 | org.eclipse.jdt.ui.importorder=\#org.junit.Assert;\#org.mockito.Mockito;\#org.fest.assertions.Assertions;\#org.fest.assertions.MapAssert;java;javax;org;com; 7 | org.eclipse.jdt.ui.ondemandthreshold=99 8 | org.eclipse.jdt.ui.staticondemandthreshold=1 9 | -------------------------------------------------------------------------------- /src/build/eclipse/org.moreunit.prefs: -------------------------------------------------------------------------------- 1 | #Fri Sep 03 12:44:49 CEST 2010 2 | eclipse.preferences.version=1 3 | org.moreunit.prefixes= 4 | org.moreunit.test_superclass=net.sf.AbstractTest 5 | org.moreunit.test_type=junit4 6 | org.moreunit.unitsourcefolder=${module}\:src/main/java\:${module}\:src/test/java\#${module}\:src/main/java\:${module}\:src/it/java 7 | org.moreunit.useprojectsettings=true 8 | -------------------------------------------------------------------------------- /trash/IZkChildListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.zk; 17 | 18 | import java.util.List; 19 | 20 | import net.sf.katta.util.KattaException; 21 | 22 | /** 23 | * An {@link IZkChildListener} can be registered at a {@link ZKClient} for 24 | * listening on zk child changes for a given path. 25 | * 26 | * Node: Also this listener re-subscribes it watch for the path on each zk event 27 | * (zk watches are one-timers) is is not guaranteed that events on the path are 28 | * missing (see http://zookeeper.wiki.sourceforge.net/ZooKeeperWatches). An 29 | * implementation of this class should take that into account. 30 | * 31 | */ 32 | public interface IZkChildListener { 33 | 34 | void handleChildChange(String parentPath, List currentChilds) throws KattaException; 35 | } 36 | -------------------------------------------------------------------------------- /trash/IZkDataListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.zk; 17 | 18 | /** 19 | * An {@link IZkDataListener} can be registered at a {@link ZKClient} for 20 | * listening on zk data changes for a given path. 21 | * 22 | * Node: Also this listener re-subscribes it watch for the path on each zk event 23 | * (zk watches are one-timers) is is not guaranteed that events on the path are 24 | * missing (see http://zookeeper.wiki.sourceforge.net/ZooKeeperWatches). An 25 | * implementation of this class should take that into account. 26 | */ 27 | import net.sf.katta.util.KattaException; 28 | 29 | import org.apache.hadoop.io.Writable; 30 | 31 | public interface IZkDataListener { 32 | 33 | void handleDataChange(String dataPath, W data) throws KattaException; 34 | 35 | void handleDataAdded(String dataPath, W data) throws KattaException; 36 | 37 | void handleDataDeleted(String dataPath) throws KattaException; 38 | 39 | W createWritable(); 40 | } 41 | -------------------------------------------------------------------------------- /trash/IZkReconnectListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.zk; 17 | 18 | import org.apache.zookeeper.Watcher.Event.KeeperState; 19 | 20 | public interface IZkReconnectListener { 21 | 22 | /** 23 | * Called when the zookeeper connection state has changed. 24 | * 25 | * @param state 26 | * The new state. 27 | * @throws Exception 28 | * On any error. 29 | */ 30 | public void handleStateChanged(KeeperState state) throws Exception; 31 | 32 | /** 33 | * Called after the zookeeper session has expired and a new session has been 34 | * created. You would have to re-create any ephemeral nodes here. 35 | * 36 | * @throws Exception 37 | * On any error. 38 | */ 39 | public void handleNewSession() throws Exception; 40 | } 41 | -------------------------------------------------------------------------------- /trash/NumberPaddingUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.text.DecimalFormat; 19 | import java.util.TreeSet; 20 | 21 | public class NumberPaddingUtil { 22 | 23 | private static final DecimalFormat FORMAT = new DecimalFormat("0000000000.####"); 24 | 25 | /** 26 | * @param number 27 | * @return the padded number as string 28 | */ 29 | public static String padding(final double number) { 30 | return FORMAT.format(number); 31 | } 32 | 33 | public static void main(final String[] args) { 34 | final TreeSet set = new TreeSet(); 35 | set.add(NumberPaddingUtil.padding(1)); 36 | set.add(NumberPaddingUtil.padding(1.01)); 37 | set.add(NumberPaddingUtil.padding(1.10)); 38 | set.add(NumberPaddingUtil.padding(1.123)); 39 | set.add(NumberPaddingUtil.padding(10.123)); 40 | set.add(NumberPaddingUtil.padding(2)); 41 | set.add(NumberPaddingUtil.padding(2.3)); 42 | set.add(NumberPaddingUtil.padding(9)); 43 | for (final Object object : set) { 44 | System.out.println(object); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /trash/SymlinkResourceLoader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.util; 17 | 18 | import java.io.File; 19 | import java.net.MalformedURLException; 20 | import java.net.URL; 21 | 22 | /** 23 | * With help of this class a file outside or inside the classpath can be made 24 | * available to the classpath under a given name. 25 | * 26 | */ 27 | public class SymlinkResourceLoader extends ClassLoader { 28 | 29 | private final String _symlink; 30 | 31 | private final URL _target; 32 | 33 | public SymlinkResourceLoader(ClassLoader parentLoader, String symlink, File target) { 34 | super(parentLoader); 35 | _symlink = symlink; 36 | try { 37 | _target = target.toURI().toURL(); 38 | } catch (MalformedURLException e) { 39 | throw new RuntimeException("could not create url out of '" + target + "'", e); 40 | } 41 | } 42 | 43 | @Override 44 | public URL getResource(String name) { 45 | if (_symlink.equals(name)) { 46 | return _target; 47 | } 48 | return super.getResource(name); 49 | } 50 | 51 | @Override 52 | protected URL findResource(String name) { 53 | if (_symlink.equals(name)) { 54 | return _target; 55 | } 56 | return super.findResource(name); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /trash/ZkEventType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.zk; 17 | 18 | import org.apache.zookeeper.Watcher; 19 | 20 | /** 21 | * To improve handling of zk events, this enum is a kind adapter for the types 22 | * of {@link Watcher.Event}. 23 | * 24 | */ 25 | @Deprecated 26 | public enum ZkEventType { 27 | 28 | NODE_CREATED, NODE_DELETED, NODE_DATA_CHANGED, NODE_CHILDREN_CHANGED; 29 | 30 | public static ZkEventType getMappedType(int zkEventType) { 31 | switch (zkEventType) { 32 | case 1: 33 | return NODE_CREATED; 34 | case 2: 35 | return NODE_DELETED; 36 | case 3: 37 | return NODE_DATA_CHANGED; 38 | case 4: 39 | return NODE_CHILDREN_CHANGED; 40 | default: 41 | throw new RuntimeException("type " + zkEventType + " not mapped"); 42 | } 43 | } 44 | 45 | public static UnsupportedOperationException newUnhandledTypeException(ZkEventType eventType) { 46 | return new UnsupportedOperationException("unhandled type " + eventType); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /trash/ZkServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.sf.katta.zk; 17 | 18 | import net.sf.katta.AbstractKattaTest; 19 | 20 | import org.apache.zookeeper.CreateMode; 21 | import org.apache.zookeeper.WatchedEvent; 22 | import org.apache.zookeeper.Watcher; 23 | import org.apache.zookeeper.ZooKeeper; 24 | import org.apache.zookeeper.ZooDefs.Ids; 25 | import org.apache.zookeeper.ZooKeeper.States; 26 | 27 | public class ZkServerTest extends AbstractKattaTest implements Watcher { 28 | 29 | public void testServer() throws Exception { 30 | stopZkServer(); 31 | final String path = "/testPath"; 32 | ZooKeeper zk = null; 33 | try { 34 | zk = new ZooKeeper(_conf.getZKServers(), _conf.getZKClientPort(), this); 35 | zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); 36 | fail("no server yet started"); 37 | } catch (final Exception e) { 38 | zk.close(); 39 | } 40 | 41 | startZkServer(); 42 | zk = new ZooKeeper(_conf.getZKServers(), _conf.getZKClientPort(), this); 43 | while (zk.getState() == States.CONNECTING) { 44 | Thread.sleep(500); 45 | } 46 | 47 | zk.create(path, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); 48 | 49 | zk.getChildren(path, true); 50 | zk.create(path + "/2", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); 51 | zk.getChildren(path, true); 52 | zk.create(path + "/3", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); 53 | zk.close(); 54 | } 55 | 56 | public void process(final WatchedEvent event) { 57 | // System.out.println("path: " + event.getPath()); 58 | // System.out.println("type: " + event.getType()); 59 | // System.out.println("state: " + event.getState()); 60 | } 61 | 62 | } 63 | --------------------------------------------------------------------------------