├── .github └── settings.yml ├── .gitignore ├── .travis.yml ├── .travis ├── gpg.asc.enc └── mvn-settings.xml ├── LICENSE.txt ├── README.md ├── deploy ├── docs ├── hank-architecture.graffle ├── hank-architecture.png ├── hank-caching.graffle ├── hank-caching.png ├── hank-fixed-length-values.graffle ├── hank-fixed-length-values.png ├── hank-screenshot-admin.jpg ├── hank-screenshot-domain-group.jpg ├── hank-screenshot-partitions.jpg ├── hank-screenshot-ring-group-1.jpg ├── hank-screenshot-ring-group-2.jpg ├── hank-screenshot-ring-groups.jpg ├── hank-screenshot-ring.jpg ├── hank-variable-length-values.graffle └── hank-variable-length-values.png ├── example_config ├── monitor_example.yaml ├── partition_server.log4j.properties ├── partition_server_example.yaml ├── ring_group_conductor.log4j.properties ├── ring_group_conductor_example.yaml ├── smart_client.log4j.properties ├── smart_client_example.yaml └── web_ui_example.yaml ├── hank-client ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── liveramp │ │ └── hank │ │ ├── client │ │ ├── Clients.java │ │ ├── ConnectionLoad.java │ │ ├── DomainAndKey.java │ │ ├── EmptyKeyException.java │ │ ├── FutureGet.java │ │ ├── GetBulkCallback.java │ │ ├── GetCallback.java │ │ ├── GetTaskRunnableIface.java │ │ ├── HankClientIface.java │ │ ├── HankSmartClient.java │ │ ├── HankSmartClientIface.java │ │ ├── HankSmartClientOptions.java │ │ ├── HostConnection.java │ │ ├── HostConnectionPool.java │ │ ├── MockHankSmartClient.java │ │ ├── NullKeyException.java │ │ └── SmartClientDaemon.java │ │ └── config │ │ ├── ClientConfigurator.java │ │ ├── EnvironmentValue.java │ │ ├── HankSmartClientConfigurator.java │ │ ├── SmartClientDaemonConfigurator.java │ │ └── yaml │ │ ├── YamlClientConfigurator.java │ │ ├── YamlHankSmartClientConfigurator.java │ │ └── YamlSmartClientDaemonConfigurator.java │ └── test │ └── java │ └── com │ └── liveramp │ └── hank │ ├── client │ ├── TestHankSmartClient.java │ ├── TestHostConnection.java │ └── TestHostConnectionPool.java │ └── config │ └── yaml │ ├── TestYamlHankSmartClientConfigurator.java │ └── TestYamlSmartClientDaemonConfigurator.java ├── hank-core ├── pom.xml └── src │ ├── main │ ├── go │ │ └── hank │ │ │ ├── GoUnusedProtection__.go │ │ │ ├── hank-consts.go │ │ │ ├── hank.go │ │ │ ├── partition_server-remote │ │ │ └── partition_server-remote.go │ │ │ └── smart_client-remote │ │ │ └── smart_client-remote.go │ ├── java │ │ └── com │ │ │ └── liveramp │ │ │ └── hank │ │ │ ├── Hank.java │ │ │ ├── compression │ │ │ ├── CompressionCodec.java │ │ │ ├── CompressionFactory.java │ │ │ ├── Compressor.java │ │ │ ├── Decompressor.java │ │ │ ├── common │ │ │ │ └── StreamCopyDecompressor.java │ │ │ ├── cueball │ │ │ │ ├── CueballCompressionCodec.java │ │ │ │ ├── GzipCueballCompressionCodec.java │ │ │ │ └── NoCueballCompressionCodec.java │ │ │ ├── deflate │ │ │ │ ├── DeflateCompressionFactory.java │ │ │ │ ├── DeflateCompressor.java │ │ │ │ └── DeflateDecompressor.java │ │ │ ├── none │ │ │ │ ├── SlowNoCompressionCompressionFactory.java │ │ │ │ ├── SlowNoCompressionCompressor.java │ │ │ │ └── SlowNoCompressionDecompressor.java │ │ │ ├── snappy │ │ │ │ ├── SnappyCompressionFactory.java │ │ │ │ ├── SnappyCompressor.java │ │ │ │ └── SnappyDecompressor.java │ │ │ └── zip │ │ │ │ ├── GzipCompressionFactory.java │ │ │ │ ├── GzipCompressor.java │ │ │ │ └── GzipDecompressor.java │ │ │ ├── config │ │ │ ├── BaseReaderConfigurator.java │ │ │ ├── CoordinatorConfigurator.java │ │ │ ├── DataDirectoriesConfigurator.java │ │ │ ├── InvalidConfigurationException.java │ │ │ ├── PartitionServerConfigurator.java │ │ │ ├── ReaderConfigurator.java │ │ │ └── yaml │ │ │ │ ├── YamlConfigurator.java │ │ │ │ └── YamlCoordinatorConfigurator.java │ │ │ ├── coordinator │ │ │ ├── AbstractDomain.java │ │ │ ├── AbstractDomainGroup.java │ │ │ ├── AbstractDomainVersion.java │ │ │ ├── AbstractHost.java │ │ │ ├── AbstractHostDomain.java │ │ │ ├── AbstractHostDomainPartition.java │ │ │ ├── AbstractRing.java │ │ │ ├── AbstractRingGroup.java │ │ │ ├── CloseCoordinatorOpportunistically.java │ │ │ ├── Coordinator.java │ │ │ ├── CoordinatorFactory.java │ │ │ ├── DataLocationChangeListener.java │ │ │ ├── Domain.java │ │ │ ├── DomainAndVersion.java │ │ │ ├── DomainGroup.java │ │ │ ├── DomainGroupListener.java │ │ │ ├── DomainGroups.java │ │ │ ├── DomainVersion.java │ │ │ ├── DomainVersionProperties.java │ │ │ ├── DomainVersionPropertiesSerialization.java │ │ │ ├── DomainVersions.java │ │ │ ├── Domains.java │ │ │ ├── Host.java │ │ │ ├── HostAddress.java │ │ │ ├── HostCommand.java │ │ │ ├── HostCommandQueueChangeListener.java │ │ │ ├── HostDomain.java │ │ │ ├── HostDomainPartition.java │ │ │ ├── HostDomains.java │ │ │ ├── HostState.java │ │ │ ├── Hosts.java │ │ │ ├── PartitionProperties.java │ │ │ ├── PartitionServerAddress.java │ │ │ ├── Ring.java │ │ │ ├── RingGroup.java │ │ │ ├── RingGroupDataLocationChangeListener.java │ │ │ ├── RingGroups.java │ │ │ ├── Rings.java │ │ │ ├── RunWithCoordinator.java │ │ │ ├── RunnableWithCoordinator.java │ │ │ ├── ServingStatus.java │ │ │ ├── ServingStatusAggregator.java │ │ │ ├── UpdateProgress.java │ │ │ ├── UpdateProgressAggregator.java │ │ │ ├── mock │ │ │ │ ├── MockCoordinator.java │ │ │ │ ├── MockDomain.java │ │ │ │ ├── MockDomainGroup.java │ │ │ │ ├── MockDomainVersion.java │ │ │ │ └── StaticMockDomain.java │ │ │ └── zk │ │ │ │ ├── DotComplete.java │ │ │ │ ├── HankWatcher.java │ │ │ │ ├── MigrationHelper.java │ │ │ │ ├── ZkDomain.java │ │ │ │ ├── ZkDomainGroup.java │ │ │ │ ├── ZkDomainVersion.java │ │ │ │ ├── ZkHost.java │ │ │ │ ├── ZkHostDomain.java │ │ │ │ ├── ZkHostDomainPartition.java │ │ │ │ ├── ZkPartitionProperties.java │ │ │ │ ├── ZkRing.java │ │ │ │ ├── ZkRingGroup.java │ │ │ │ └── ZooKeeperCoordinator.java │ │ │ ├── generated │ │ │ ├── ClientMetadata.java │ │ │ ├── ConnectedServerMetadata.java │ │ │ ├── DomainGroupMetadata.java │ │ │ ├── DomainMetadata.java │ │ │ ├── DomainStatisticsSummary.java │ │ │ ├── DomainVersionMetadata.java │ │ │ ├── HankBulkResponse.java │ │ │ ├── HankException.java │ │ │ ├── HankResponse.java │ │ │ ├── HostAssignmentsMetadata.java │ │ │ ├── HostDomainMetadata.java │ │ │ ├── HostDomainPartitionMetadata.java │ │ │ ├── HostMetadata.java │ │ │ ├── LatencySampleSummary.java │ │ │ ├── PartitionMetadata.java │ │ │ ├── PartitionServer.java │ │ │ ├── RuntimeStatisticsSummary.java │ │ │ ├── SmartClient.java │ │ │ └── StatisticsMetadata.java │ │ │ ├── hasher │ │ │ ├── Hasher.java │ │ │ ├── IdentityHasher.java │ │ │ ├── LeftPaddedIdentityHasher.java │ │ │ └── Murmur64Hasher.java │ │ │ ├── partition_assigner │ │ │ ├── AbstractMappingPartitionAssigner.java │ │ │ ├── ModPartitionAssigner.java │ │ │ ├── PartitionAssigner.java │ │ │ └── RendezVousPartitionAssigner.java │ │ │ ├── partition_server │ │ │ ├── DiskPartitionAssignment.java │ │ │ ├── DoublePopulationStatisticsAggregator.java │ │ │ ├── FilesystemStatisticsAggregator.java │ │ │ ├── IfaceWithShutdown.java │ │ │ ├── PartitionAccessorRuntimeStatistics.java │ │ │ ├── PartitionUpdateTaskStatistics.java │ │ │ └── RuntimeStatisticsAggregator.java │ │ │ ├── partitioner │ │ │ ├── ConstantPartitioner.java │ │ │ ├── HashCodePartitioner.java │ │ │ ├── Murmur64Partitioner.java │ │ │ └── Partitioner.java │ │ │ ├── ring_group_conductor │ │ │ └── RingGroupConductorMode.java │ │ │ ├── storage │ │ │ ├── CacheStatistics.java │ │ │ ├── Compactor.java │ │ │ ├── Deleter.java │ │ │ ├── FileOpsUtil.java │ │ │ ├── LocalPartitionRemoteFileOps.java │ │ │ ├── PartitionFileLocalPath.java │ │ │ ├── PartitionRemoteFileOps.java │ │ │ ├── PartitionRemoteFileOpsFactory.java │ │ │ ├── PartitionUpdater.java │ │ │ ├── Reader.java │ │ │ ├── ReaderResult.java │ │ │ ├── RemoteDomainCleaner.java │ │ │ ├── RemoteDomainVersionDeleter.java │ │ │ ├── StorageEngine.java │ │ │ ├── StorageEngineFactory.java │ │ │ ├── Writer.java │ │ │ ├── cueball │ │ │ │ ├── Cueball.java │ │ │ │ ├── CueballDeleter.java │ │ │ │ ├── CueballFilePath.java │ │ │ │ ├── CueballMerger.java │ │ │ │ ├── CueballPartitionUpdater.java │ │ │ │ ├── CueballReader.java │ │ │ │ ├── CueballRemoteDomainCleaner.java │ │ │ │ ├── CueballRemoteDomainVersionDeleter.java │ │ │ │ ├── CueballStreamBuffer.java │ │ │ │ ├── CueballStreamBufferMergeSort.java │ │ │ │ ├── CueballUpdatePlanner.java │ │ │ │ ├── CueballWriter.java │ │ │ │ ├── Footer.java │ │ │ │ ├── HashPrefixCalculator.java │ │ │ │ ├── ICueballMerger.java │ │ │ │ ├── ICueballStreamBufferMergeSortFactory.java │ │ │ │ ├── IKeyFileStreamBufferMergeSort.java │ │ │ │ ├── KeyHashAndValueAndStreamIndex.java │ │ │ │ ├── TestDomainGenerator.java │ │ │ │ └── ValueTransformer.java │ │ │ ├── curly │ │ │ │ ├── AbstractCurlyPartitionUpdater.java │ │ │ │ ├── Curly.java │ │ │ │ ├── CurlyCompactingMerger.java │ │ │ │ ├── CurlyCompactor.java │ │ │ │ ├── CurlyDeleter.java │ │ │ │ ├── CurlyFastPartitionUpdater.java │ │ │ │ ├── CurlyFilePath.java │ │ │ │ ├── CurlyMerger.java │ │ │ │ ├── CurlyReader.java │ │ │ │ ├── CurlyRemoteDomainCleaner.java │ │ │ │ ├── CurlyRemoteDomainVersionDeleter.java │ │ │ │ ├── CurlyUpdatePlanner.java │ │ │ │ ├── CurlyWriter.java │ │ │ │ ├── ICurlyCompactingMerger.java │ │ │ │ ├── ICurlyMerger.java │ │ │ │ ├── ICurlyReader.java │ │ │ │ ├── ICurlyReaderFactory.java │ │ │ │ └── TestDomainGenerator.java │ │ │ ├── echo │ │ │ │ ├── Echo.java │ │ │ │ ├── EchoDeleter.java │ │ │ │ ├── EchoReader.java │ │ │ │ └── EchoUpdater.java │ │ │ └── incremental │ │ │ │ ├── IncrementalDomainVersionProperties.java │ │ │ │ ├── IncrementalDomainVersionPropertiesHelper.java │ │ │ │ ├── IncrementalPartitionUpdater.java │ │ │ │ ├── IncrementalRemoteDomainCleaner.java │ │ │ │ ├── IncrementalStorageEngine.java │ │ │ │ ├── IncrementalUpdatePlan.java │ │ │ │ ├── IncrementalUpdatePlanner.java │ │ │ │ ├── MockIncrementalPartitionUpdater.java │ │ │ │ └── MockIncrementalUpdatePlanner.java │ │ │ ├── test │ │ │ ├── BaseTestCase.java │ │ │ ├── CoreConfigFixtures.java │ │ │ ├── ZkMockCoordinatorTestCase.java │ │ │ ├── ZkTestCase.java │ │ │ ├── coordinator │ │ │ │ ├── MockHost.java │ │ │ │ ├── MockHostDomain.java │ │ │ │ ├── MockHostDomainPartition.java │ │ │ │ ├── MockRing.java │ │ │ │ └── MockRingGroup.java │ │ │ ├── hasher │ │ │ │ └── MapHasher.java │ │ │ └── partitioner │ │ │ │ └── MapPartitioner.java │ │ │ ├── util │ │ │ ├── AtomicLongCollection.java │ │ │ ├── CliUtils.java │ │ │ ├── CommandLineChecker.java │ │ │ ├── DurationAggregator.java │ │ │ ├── EncodingHelper.java │ │ │ ├── ExponentialBackoff.java │ │ │ ├── FormatUtils.java │ │ │ ├── FsUtils.java │ │ │ ├── HankResponseMemoryUsageEstimator.java │ │ │ ├── HankTimer.java │ │ │ ├── HankTimerEventAggregator.java │ │ │ ├── IOStreamUtils.java │ │ │ ├── LocalHostUtils.java │ │ │ ├── ReverseComparator.java │ │ │ ├── SynchronizedCache.java │ │ │ ├── SynchronizedMemoryBoundCache.java │ │ │ ├── SynchronizedMemoryBoundCacheExpiring.java │ │ │ ├── UnsafeByteArrayOutputStream.java │ │ │ └── UpdateStatisticsRunnable.java │ │ │ └── zookeeper │ │ │ ├── NodeCreationBarrier.java │ │ │ ├── WatchedBoolean.java │ │ │ ├── WatchedBytes.java │ │ │ ├── WatchedEnum.java │ │ │ ├── WatchedInt.java │ │ │ ├── WatchedLong.java │ │ │ ├── WatchedMap.java │ │ │ ├── WatchedMapListener.java │ │ │ ├── WatchedNode.java │ │ │ ├── WatchedNodeListener.java │ │ │ ├── WatchedNodeUpdater.java │ │ │ ├── WatchedNodeUpdaterWithReturnValue.java │ │ │ ├── WatchedNodeUpdaterWithReturnValueImpl.java │ │ │ ├── WatchedString.java │ │ │ ├── WatchedThriftNode.java │ │ │ ├── ZkCli.java │ │ │ ├── ZkPath.java │ │ │ ├── ZooKeeperConnection.java │ │ │ └── ZooKeeperPlus.java │ └── thrift │ │ └── hank.thrift │ └── test │ └── java │ └── com │ └── liveramp │ └── hank │ ├── config │ └── yaml │ │ └── TestYamlConfigurator.java │ ├── coordinator │ ├── TestAbstractRing.java │ └── zk │ │ ├── MockHostCommandQueueChangeListener.java │ │ ├── MockHostStateChangeListener.java │ │ ├── TestZkDomain.java │ │ ├── TestZkDomainGroup.java │ │ ├── TestZkDomainVersion.java │ │ ├── TestZkHost.java │ │ ├── TestZkPartitionProperties.java │ │ ├── TestZkRing.java │ │ ├── TestZkRingGroup.java │ │ └── TestZooKeeperCoordinator.java │ ├── hasher │ ├── TestLeftPaddedIdentityHasher.java │ └── TestMurmur64Hasher.java │ ├── partition_assigner │ └── TestRendezVousPartitionAssigner.java │ ├── storage │ ├── constant │ │ └── ConstantStorageEngine.java │ └── echo │ │ └── TestEchoReader.java │ ├── util │ └── TestEncodingHelper.java │ └── zookeeper │ ├── TestWatchedInt.java │ ├── TestWatchedLong.java │ ├── TestWatchedMap.java │ ├── TestZkPath.java │ └── TestZooKeeperPlus.java ├── hank-server ├── bin │ └── hank-server ├── conf │ └── .gitignore ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── liveramp │ │ └── hank │ │ ├── cascading │ │ ├── CascadingDomainBuilder.java │ │ ├── DomainBuilderAssembly.java │ │ ├── DomainBuilderTap.java │ │ ├── FlowConnectorFactory.java │ │ ├── HadoopFlowConnectorFactory.java │ │ └── PartitionMarkerTap.java │ │ ├── config │ │ ├── RingGroupConductorConfigurator.java │ │ ├── RingGroupConfiguredDomain.java │ │ ├── SimpleDataDirectoriesConfigurator.java │ │ └── yaml │ │ │ ├── YamlPartitionServerConfigurator.java │ │ │ └── YamlRingGroupConductorConfigurator.java │ │ ├── hadoop │ │ ├── AbstractHadoopDomainBuilder.java │ │ ├── DomainBuilderAbstractOutputFormat.java │ │ ├── DomainBuilderBaseOutputFormat.java │ │ ├── DomainBuilderDefaultOutputFormat.java │ │ ├── DomainBuilderMapper.java │ │ ├── DomainBuilderMapperDefault.java │ │ ├── DomainBuilderOutputCommitter.java │ │ ├── DomainBuilderPartitioner.java │ │ ├── DomainBuilderProperties.java │ │ ├── DomainBuilderReducer.java │ │ ├── DomainCompactorOutputFormat.java │ │ ├── DomainCompactorProperties.java │ │ ├── DomainVersionNumberAndNumPartitions.java │ │ ├── HadoopDomainBuilder.java │ │ ├── HadoopDomainCompactor.java │ │ ├── KeyAndPartitionWritable.java │ │ ├── KeyAndPartitionWritableComparable.java │ │ ├── KeyValuePair.java │ │ ├── OutputCollectorWriter.java │ │ ├── PartitionIntWritable.java │ │ ├── ValueWritable.java │ │ └── test │ │ │ └── MapStorageEngineCoordinator.java │ │ ├── loadtest │ │ └── RandomSaturator.java │ │ ├── partition_server │ │ ├── DomainAccessor.java │ │ ├── IUpdateManager.java │ │ ├── PartitionAccessor.java │ │ ├── PartitionServer.java │ │ ├── PartitionServerHandler.java │ │ └── UpdateManager.java │ │ ├── performance │ │ ├── PerformanceTestCueball.java │ │ └── RandomReadPerformance.java │ │ ├── ring_group_conductor │ │ ├── HostReplicaStatus.java │ │ ├── LiveReplicaStatus.java │ │ ├── PartitionUtils.java │ │ ├── RingGroupAutoconfigureTransitionFunction.java │ │ ├── RingGroupConductor.java │ │ ├── RingGroupTransitionFunction.java │ │ └── RingGroupUpdateTransitionFunctionImpl.java │ │ ├── storage │ │ ├── HdfsPartitionRemoteFileOps.java │ │ ├── NoOpPartitionRemoteFileOps.java │ │ ├── RemoteDomainVersionDeletionHelper.java │ │ ├── map │ │ │ ├── MapReader.java │ │ │ ├── MapStorageEngine.java │ │ │ └── MapWriter.java │ │ └── mock │ │ │ ├── MockDeleter.java │ │ │ ├── MockReader.java │ │ │ └── MockStorageEngine.java │ │ └── test │ │ └── ConfigFixtures.java │ └── test │ └── java │ └── com │ └── liveramp │ └── hank │ ├── IntegrationTest.java │ ├── cascading │ └── TestCascadingDomainBuilder.java │ ├── compression │ └── cueball │ │ └── TestGzipCueballCompressionCodec.java │ ├── config │ └── yaml │ │ ├── TestYamlPartitionServerConfigurator.java │ │ └── TestYamlRingGroupConductorConfigurator.java │ ├── fixtures │ ├── PartitionServerRunnable.java │ └── RingGroupConductorRunnable.java │ ├── hadoop │ ├── DomainBuilderEmptyOutputFormat.java │ ├── HadoopTestCase.java │ ├── IntStringKeyStorageEngineCoordinator.java │ ├── TestHadoopDomainBuilder.java │ ├── TestHadoopDomainCompactor.java │ └── test │ │ └── TestMapStorageEngineCoordinator.java │ ├── partition_server │ ├── MockPartitionServerConfigurator.java │ ├── MockPartitionUpdater.java │ ├── MockUpdateManager.java │ ├── TestDomainAccessor.java │ ├── TestDoublePopulationStatisticsAggregator.java │ ├── TestPartitionServer.java │ ├── TestPartitionServerHandler.java │ └── TestUpdateManager.java │ ├── ring_group_conductor │ ├── TestRingGroupAutconfigure.java │ ├── TestRingGroupConductor.java │ └── TestRingGroupUpdateTransitionFunctionImpl.java │ └── storage │ ├── TestHdfsPartitionRemoteFileOps.java │ ├── cueball │ ├── AbstractCueballTest.java │ ├── MockCueballMerger.java │ ├── TestCueballDeleter.java │ ├── TestCueballFactory.java │ ├── TestCueballMerger.java │ ├── TestCueballPartitionUpdater.java │ ├── TestCueballReader.java │ ├── TestCueballRemoteDomainVersionDeleter.java │ ├── TestCueballStreamBuffer.java │ ├── TestCueballWriter.java │ ├── TestFooter.java │ └── TestHashPrefixCalculator.java │ ├── curly │ ├── AbstractCurlyTestBase.java │ ├── MockCurlyMerger.java │ ├── TestAbstractCurlyPartitionUpdater.java │ ├── TestCurlyCompactingMerger.java │ ├── TestCurlyCompactor.java │ ├── TestCurlyDeleter.java │ ├── TestCurlyFactory.java │ ├── TestCurlyFastPartitionUpdater.java │ ├── TestCurlyMerger.java │ ├── TestCurlyReader.java │ ├── TestCurlyRemoteDomainVersionDeleter.java │ └── TestCurlyWriter.java │ └── incremental │ ├── IncrementalPartitionUpdaterTestCase.java │ ├── TestIncrementalPartitionUpdater.java │ └── TestIncrementalRemoteDomainCleaner.java ├── hank-ui ├── bin │ └── hank-ui ├── conf │ └── .gitignore ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── liveramp │ │ └── hank │ │ ├── config │ │ ├── MockMonitorConfigurator.java │ │ ├── MonitorConfigurator.java │ │ └── yaml │ │ │ └── YamlMonitorConfigurator.java │ │ ├── monitor │ │ ├── HostMonitor.java │ │ ├── Monitor.java │ │ ├── RingGroupMonitor.java │ │ ├── RingMonitor.java │ │ ├── notification │ │ │ ├── AbstractNotification.java │ │ │ ├── DomainGroupChangeNotification.java │ │ │ ├── EmailNotificationFormatter.java │ │ │ ├── HostStateNotification.java │ │ │ ├── NotificationFormatter.java │ │ │ ├── RingGroupConductorModeNotification.java │ │ │ └── StringNotification.java │ │ └── notifier │ │ │ ├── AbstractNotifierFactory.java │ │ │ ├── Notification.java │ │ │ ├── Notifier.java │ │ │ ├── NotifierFactory.java │ │ │ ├── db │ │ │ ├── ISQLNotifierConfiguration.java │ │ │ ├── SQLEnvironmentsNotifierFactory.java │ │ │ ├── SQLNotifier.java │ │ │ ├── SQLNotifierConfiguration.java │ │ │ └── SQLNotifierFactory.java │ │ │ ├── email │ │ │ ├── EmailSummaryNotifier.java │ │ │ └── EmailSummaryNotifierFactory.java │ │ │ └── mock │ │ │ ├── MockNotifier.java │ │ │ └── MockNotifierFactory.java │ │ └── ui │ │ ├── ClientCache.java │ │ ├── ClientMetadataComparator.java │ │ ├── HankApiHelper.java │ │ ├── HankApiServlet.java │ │ ├── IClientCache.java │ │ ├── URLEnc.java │ │ ├── UiUtils.java │ │ ├── WebUiServer.java │ │ ├── _footer.jsp │ │ ├── _head.jsp │ │ ├── _top_nav.jsp │ │ ├── admin.jsp │ │ ├── admin_host.jsp │ │ ├── admin_ring.jsp │ │ ├── admin_ring_group.jsp │ │ ├── controllers │ │ ├── Action.java │ │ ├── Controller.java │ │ ├── DomainController.java │ │ ├── DomainGroupController.java │ │ ├── HostController.java │ │ ├── RingController.java │ │ └── RingGroupController.java │ │ ├── css │ │ └── global.css │ │ ├── domain.jsp │ │ ├── domain_group.jsp │ │ ├── domain_groups.jsp │ │ ├── domains.jsp │ │ ├── host.jsp │ │ ├── index.jsp │ │ ├── new_domain.jsp │ │ ├── ring.jsp │ │ ├── ring_group.jsp │ │ └── ring_groups.jsp │ └── test │ └── java │ └── com │ └── liveramp │ └── hank │ ├── config │ └── yaml │ │ └── TestYamlMonitorConfigurator.java │ ├── monitor │ ├── TestHostMonitor.java │ ├── TestRingGroupMonitor.java │ └── notifier │ │ └── db │ │ ├── SQLEnvironmentsNotifierFactoryTest.java │ │ └── SQLNotifierFactoryTest.java │ └── ui │ ├── TestHankApiHelper.java │ ├── TestHankApiServlet.java │ ├── TestWebUiServer.java │ └── WebUiServerTester.java ├── pom.xml └── src └── assembly ├── dist.xml └── jobjar.xml /.github/settings.yml: -------------------------------------------------------------------------------- 1 | repository: 2 | default_branch: master 3 | branches: 4 | - name: master 5 | protection: 6 | required_status_checks: null 7 | enforce_admins: false 8 | required_pull_request_reviews: 9 | dismissal_restrictions: {} 10 | require_code_owner_reviews: true 11 | required_approving_review_count: 1 12 | restrictions: null 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | .idea 3 | target 4 | **/*.iml 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: oraclejdk8 3 | notifications: 4 | email: false 5 | env: 6 | global: 7 | - secure: IrZVQtT/ex0pT1F1ZsxHtN0haO4hdPcfAnrXT7+OshcweSviYE3zDVzP/uSXB0/CIg4JlMqlQOZrl6Kud3S40QHFVlFeR7nhcStBJ1nvie7Rk94p/n4TCTU/Erv8U9q1MUbKzLMc6xtui9aWHf7vGcvnf3kaFNGDDMPb55pwCuw= 8 | - secure: OyIgeh08HNgJ5Tz8NyfumR4/IbmHobpzgpPdzYKgHG93hFAvfsgL6KfqiK+90tjaztBkXlqn442IrdeKYqFB0IqRPLlzSODqAz1Bi9ybXpE8SyQyPbyRK7kzvlf8Q02DgIspqy7TYzBgDwWs6ee7UpbT9NptKHJQK+sR5/rLQ9k= 9 | - secure: eKIi3wVE5nMzq9dlcgtwetn8pjr5ZQ1bHHRtmGM7hM5ncvzyIb4ul7CXBTn4XDD4RGDLv1ib0viHBOOrBntZRwofHvNYD2dFA3374xRz1AYC+pmcsaqSzk8yq+pwITgy2W1AHguvH7VXR+N1/8MhjyKbGKHlPiTboG87OJkEGqw= 10 | - secure: T/RSCydqVci8RFxFO7oRPU3Sd6OyR2wck7w+v+Q0WNJOnVNMlldcXm2hb7YtKipvCH+iYsUYgAy7s6ZptR09RUofgRwss/tTV26DgcRNYcJGOqRZPEJWe29Q6DLpGiCEKv8BWSbYyhwb3o11qCXG/C5wXdxZh4tvF+qYmIoK57E= 11 | before_deploy: 12 | - openssl aes-256-cbc -K $encrypted_cd343739e94e_key -iv $encrypted_cd343739e94e_iv 13 | -in .travis/gpg.asc.enc -out .travis/gpg.asc -d 14 | deploy: 15 | skip_cleanup: true 16 | on: 17 | branch: master 18 | provider: script 19 | script: ./deploy 20 | -------------------------------------------------------------------------------- /.travis/gpg.asc.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/.travis/gpg.asc.enc -------------------------------------------------------------------------------- /.travis/mvn-settings.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 10 | ossrh 11 | ${env.OSSRH_JIRA_USERNAME} 12 | ${env.OSSRH_JIRA_PASSWORD} 13 | 14 | 15 | 16 | 17 | 18 | ossrh 19 | 20 | true 21 | 22 | 23 | gpg 24 | ${env.GPG_KEY_NAME} 25 | ${env.GPG_PASSPHRASE} 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2014 LiveRamp 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /deploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # expects variables to be set: 3 | # - OSSRH_JIRA_USERNAME 4 | # - OSSRH_JIRA_PASSWORD 5 | # - GPG_KEY_NAME 6 | # - GPG_PASSPHRASE 7 | # expects file to exist: 8 | # - .travis/gpg.asc 9 | 10 | set -e 11 | 12 | # Check the variables are set 13 | if [ -z "$OSSRH_JIRA_USERNAME" ]; then 14 | echo "missing environment value: OSSRH_JIRA_USERNAME" >&2 15 | exit 1 16 | fi 17 | 18 | if [ -z "$OSSRH_JIRA_PASSWORD" ]; then 19 | echo "missing environment value: OSSRH_JIRA_PASSWORD" >&2 20 | exit 1 21 | fi 22 | 23 | if [ -z "$GPG_KEY_NAME" ]; then 24 | echo "missing environment value: GPG_KEY_NAME" >&2 25 | exit 1 26 | fi 27 | 28 | if [ -z "$GPG_PASSPHRASE" ]; then 29 | echo "missing environment value: GPG_PASSPHRASE" >&2 30 | exit 1 31 | fi 32 | 33 | # Prepare the local keyring (requires travis to have decrypted the file 34 | # beforehand) 35 | gpg --fast-import .travis/gpg.asc 36 | 37 | # Run the maven deploy steps 38 | mvn deploy -P publish -DskipTests=true --settings "${TRAVIS_BUILD_DIR}/.travis/mvn-settings.xml" 39 | -------------------------------------------------------------------------------- /docs/hank-architecture.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-architecture.graffle -------------------------------------------------------------------------------- /docs/hank-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-architecture.png -------------------------------------------------------------------------------- /docs/hank-caching.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-caching.graffle -------------------------------------------------------------------------------- /docs/hank-caching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-caching.png -------------------------------------------------------------------------------- /docs/hank-fixed-length-values.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-fixed-length-values.graffle -------------------------------------------------------------------------------- /docs/hank-fixed-length-values.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-fixed-length-values.png -------------------------------------------------------------------------------- /docs/hank-screenshot-admin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-admin.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-domain-group.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-domain-group.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-partitions.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-partitions.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-ring-group-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-ring-group-1.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-ring-group-2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-ring-group-2.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-ring-groups.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-ring-groups.jpg -------------------------------------------------------------------------------- /docs/hank-screenshot-ring.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-screenshot-ring.jpg -------------------------------------------------------------------------------- /docs/hank-variable-length-values.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-variable-length-values.graffle -------------------------------------------------------------------------------- /docs/hank-variable-length-values.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/docs/hank-variable-length-values.png -------------------------------------------------------------------------------- /example_config/monitor_example.yaml: -------------------------------------------------------------------------------- 1 | monitor: 2 | 3 | web_ui_url: http://hank_web_ui_url 4 | global_notifier_factory: com.liveramp.hank.monitor.notifier.EmailSummaryNotifierFactory 5 | global_notifier_configuration: 6 | email_notification_targets: email1@domain, email2@domain 7 | smtp_host: smtp_server_address 8 | 9 | ring_group_notifiers: 10 | ring_group_name_1: 11 | factory: com.liveramp.hank.monitor.notifier.EmailSummaryNotifierFactory 12 | configuration: 13 | email_notification_targets: email1@domain, email2@domain 14 | smtp_host: smtp_server_address 15 | -------------------------------------------------------------------------------- /example_config/partition_server.log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | hank.partition_server.root.logger=INFO,DRFA 3 | hank.log.dir=log 4 | hank.partition_server.log.file=partition_server.log 5 | 6 | # Define the root logger to the system property "hbase.root.logger". 7 | log4j.rootLogger=${hank.partition_server.root.logger} 8 | 9 | # Logging Threshold 10 | log4j.threshhold=INFO 11 | 12 | # 13 | # Daily Rolling File Appender 14 | # 15 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 16 | log4j.appender.DRFA.File=${hank.log.dir}/${hank.partition_server.log.file} 17 | 18 | # Rollver at midnight 19 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 20 | 21 | # 30-day backup 22 | #log4j.appender.DRFA.MaxBackupIndex=30 23 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 24 | 25 | # Pattern format: Date LogLevel LoggerName LogMessage 26 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 27 | 28 | # Debugging Pattern format 29 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 30 | 31 | # console is set to be a ConsoleAppender. 32 | log4j.appender.console=org.apache.log4j.ConsoleAppender 33 | 34 | # A1 uses PatternLayout. 35 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 36 | log4j.appender.console.layout.ConversionPattern=%d [%t] %-5p %l: %m%n 37 | 38 | # Custom Logging levels 39 | # If we don't do this, then Hadoop will be noisy. 40 | log4j.logger.org.apache.hadoop=INFO 41 | -------------------------------------------------------------------------------- /example_config/partition_server_example.yaml: -------------------------------------------------------------------------------- 1 | partition_server: 2 | # add the directories that your partition server should use to store/read data 3 | # NOTE: don't change these once you've started using them, since it will mess 4 | # up the hash/mod partitioning over your disks. (unless you're an expert.) 5 | local_data_dirs: 6 | - /path/to/some/data 7 | service_port: 12345 8 | # change this to the name of the ring group this partition server should join 9 | ring_group_name: rg1 10 | partition_server_daemon: 11 | num_worker_threads: 256 12 | num_concurrent_get_bulk_tasks: 256 13 | get_bulk_task_size: 64 14 | get_timer_aggregator_window: 1000 15 | update_daemon: 16 | num_concurrent_updates: 1 17 | coordinator: 18 | factory: com.liveramp.hank.coordinator.zk.ZooKeeperCoordinator$Factory 19 | options: 20 | # a ZooKeeper connection string that identifies your quorum 21 | connect_string: localhost:2181 22 | session_timeout: 2000 23 | domains_root: /hank/domains 24 | domain_groups_root: /hank/domain_groups 25 | ring_groups_root: /hank/ring_groups 26 | -------------------------------------------------------------------------------- /example_config/ring_group_conductor.log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | hank.ring_group_conductor.root.logger=INFO,DRFA 3 | hank.log.dir=log 4 | hank.ring_group_conductor.log.file=ring_group_conductor.log 5 | 6 | # Define the root logger to the system property "hbase.root.logger". 7 | log4j.rootLogger=${hank.ring_group_conductor.root.logger} 8 | 9 | # Logging Threshold 10 | log4j.threshhold=INFO 11 | 12 | # 13 | # Daily Rolling File Appender 14 | # 15 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 16 | log4j.appender.DRFA.File=${hank.log.dir}/${hank.ring_group_conductor.log.file} 17 | 18 | # Rollver at midnight 19 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 20 | 21 | # 30-day backup 22 | #log4j.appender.DRFA.MaxBackupIndex=30 23 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 24 | 25 | # Pattern format: Date LogLevel LoggerName LogMessage 26 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 27 | 28 | # Debugging Pattern format 29 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 30 | 31 | # console is set to be a ConsoleAppender. 32 | log4j.appender.console=org.apache.log4j.ConsoleAppender 33 | 34 | # A1 uses PatternLayout. 35 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 36 | log4j.appender.console.layout.ConversionPattern=%d [%t] %-5p %l: %m%n 37 | 38 | # Custom Logging levels 39 | # If we don't do this, then Hadoop will be noisy. 40 | log4j.logger.org.apache.hadoop=INFO 41 | -------------------------------------------------------------------------------- /example_config/ring_group_conductor_example.yaml: -------------------------------------------------------------------------------- 1 | ring_group_conductor: 2 | # replace this with the name of the ring group you want the conductor to manage 3 | ring_group_name: myRingGroup 4 | # low values affect CPU usage. you probably don't need to touch this. 5 | sleep_interval: 1000 6 | initial_mode: INACTIVE 7 | coordinator: 8 | factory: com.liveramp.hank.coordinator.zk.ZooKeeperCoordinator$Factory 9 | options: 10 | # a ZooKeeper connection string that identifies your quorum 11 | connect_string: localhost:2181 12 | session_timeout: 2000 13 | domains_root: /hank/domains 14 | domain_groups_root: /hank/domain_groups 15 | ring_groups_root: /hank/ring_groups 16 | -------------------------------------------------------------------------------- /example_config/smart_client.log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | hank.smart_client.root.logger=INFO,DRFA 3 | hank.log.dir=log 4 | hank.smart_client.log.file=smart_client.log 5 | 6 | # Define the root logger to the system property "hbase.root.logger". 7 | log4j.rootLogger=${hank.smart_client.root.logger} 8 | 9 | # Logging Threshold 10 | log4j.threshhold=INFO 11 | 12 | # 13 | # Daily Rolling File Appender 14 | # 15 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 16 | log4j.appender.DRFA.File=${hank.log.dir}/${hank.smart_client.log.file} 17 | 18 | # Rollver at midnight 19 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 20 | 21 | # 30-day backup 22 | #log4j.appender.DRFA.MaxBackupIndex=30 23 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 24 | 25 | # Pattern format: Date LogLevel LoggerName LogMessage 26 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 27 | 28 | # Debugging Pattern format 29 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 30 | 31 | # console is set to be a ConsoleAppender. 32 | log4j.appender.console=org.apache.log4j.ConsoleAppender 33 | 34 | # A1 uses PatternLayout. 35 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 36 | log4j.appender.console.layout.ConversionPattern=%d [%t] %-5p %l: %m%n 37 | 38 | # Custom Logging levels 39 | # If we don't do this, then Hadoop will be noisy. 40 | log4j.logger.org.apache.hadoop=INFO 41 | -------------------------------------------------------------------------------- /example_config/smart_client_example.yaml: -------------------------------------------------------------------------------- 1 | smart_client: 2 | # this can be set to any free port. you'll give this to the thrift dumb 3 | # client in order to connect 4 | service_port: 12345 5 | # you can use this value to adjust how many concurrent requests will be 6 | # executed at once. 7 | num_worker_threads: 256 8 | # the name of the ring group that this smart client will serve. this smart 9 | # client won't be able to serve domains from any other ring group. 10 | ring_group_name: ring_group_name 11 | coordinator: 12 | factory: com.liveramp.hank.coordinator.zk.ZooKeeperCoordinator$Factory 13 | options: 14 | # a ZooKeeper connection string that identifies your quorum 15 | connect_string: localhost:2181 16 | session_timeout: 2000 17 | domains_root: /hank/domains 18 | domain_groups_root: /hank/domain_groups 19 | ring_groups_root: /hank/ring_groups 20 | -------------------------------------------------------------------------------- /example_config/web_ui_example.yaml: -------------------------------------------------------------------------------- 1 | coordinator: 2 | factory: com.liveramp.hank.coordinator.zk.ZooKeeperCoordinator$Factory 3 | options: 4 | # a ZooKeeper connection string that identifies your quorum 5 | connect_string: localhost:2181 6 | session_timeout: 2000 7 | domains_root: /hank/domains 8 | domain_groups_root: /hank/domain_groups 9 | ring_groups_root: /hank/ring_groups 10 | -------------------------------------------------------------------------------- /hank-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.liveramp.hank 5 | hank-client 6 | 7 | 1.1-SNAPSHOT 8 | 9 | 10 | UTF-8 11 | 12 | 13 | 14 | com.liveramp 15 | hank 16 | 1.1-SNAPSHOT 17 | 18 | 19 | 20 | 21 | 22 | ${project.groupId} 23 | hank-core 24 | ${project.version} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | src/main/java/ 34 | 35 | 36 | 37 | 38 | 39 | src/test/java/ 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/Clients.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | import com.liveramp.hank.Hank; 20 | import com.liveramp.hank.generated.ClientMetadata; 21 | import com.liveramp.hank.util.LocalHostUtils; 22 | 23 | import java.net.UnknownHostException; 24 | 25 | public class Clients { 26 | 27 | private Clients() { 28 | } 29 | 30 | public static ClientMetadata getClientMetadata(HankClientIface client) { 31 | String hostName; 32 | try { 33 | hostName = LocalHostUtils.getHostName(); 34 | } catch (UnknownHostException e) { 35 | hostName = "unknown"; 36 | } 37 | return new ClientMetadata( 38 | hostName, 39 | System.currentTimeMillis(), 40 | client.getClass().getName(), 41 | Hank.getGitCommit()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/ConnectionLoad.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | class ConnectionLoad { 20 | 21 | private int numConnections; 22 | private int numConnectionsLocked; 23 | 24 | ConnectionLoad(int numConnections, int numConnectionsLocked) { 25 | this.numConnections = numConnections; 26 | this.numConnectionsLocked = numConnectionsLocked; 27 | } 28 | 29 | public ConnectionLoad() { 30 | this.numConnections = 0; 31 | this.numConnectionsLocked = 0; 32 | } 33 | 34 | public int getNumConnections() { 35 | return numConnections; 36 | } 37 | 38 | public int getNumConnectionsLocked() { 39 | return numConnectionsLocked; 40 | } 41 | 42 | public void aggregate(ConnectionLoad other) { 43 | this.numConnections += other.numConnections; 44 | this.numConnectionsLocked += other.numConnectionsLocked; 45 | } 46 | 47 | // Return connection load as a percentage 48 | public double getLoad() { 49 | return ((double) numConnectionsLocked / (double) numConnections) * 100; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/EmptyKeyException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | public class EmptyKeyException extends RuntimeException { 20 | } 21 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/GetBulkCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.client; 17 | 18 | import com.liveramp.hank.generated.HankBulkResponse; 19 | 20 | public interface GetBulkCallback { 21 | 22 | public void onComplete(HankBulkResponse response); 23 | } 24 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/GetCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | import com.liveramp.hank.generated.HankResponse; 20 | 21 | public interface GetCallback { 22 | 23 | public abstract void onComplete(HankResponse response); 24 | } 25 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/GetTaskRunnableIface.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | import com.liveramp.hank.generated.HankResponse; 20 | 21 | interface GetTaskRunnableIface extends Runnable { 22 | 23 | public HankResponse getResponse(); 24 | } 25 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/HankClientIface.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | public interface HankClientIface { 20 | } 21 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/HankSmartClientIface.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.client; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.List; 5 | 6 | import com.liveramp.hank.generated.HankBulkResponse; 7 | import com.liveramp.hank.generated.HankResponse; 8 | import com.liveramp.hank.generated.SmartClient; 9 | 10 | public interface HankSmartClientIface extends HankClientIface, SmartClient.Iface { 11 | 12 | public HankResponse get(String domain_name, ByteBuffer key); 13 | 14 | public HankBulkResponse getBulk(String domain_name, List keys); 15 | 16 | public FutureGet concurrentGet(String domainName, ByteBuffer key); 17 | 18 | public List concurrentGet(String domainName, List key); 19 | 20 | public abstract void stop(); 21 | } 22 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/client/NullKeyException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.client; 18 | 19 | class NullKeyException extends RuntimeException { 20 | } 21 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/config/ClientConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.config; 17 | 18 | 19 | public interface ClientConfigurator extends CoordinatorConfigurator { 20 | } 21 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/config/EnvironmentValue.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.config; 2 | 3 | public class EnvironmentValue { 4 | 5 | private final String key; 6 | private final String value; 7 | 8 | public EnvironmentValue(String key, String value) { 9 | this.key = key; 10 | this.value = value; 11 | } 12 | 13 | public String getKey() { 14 | return key; 15 | } 16 | 17 | public String getValue() { 18 | return value; 19 | } 20 | 21 | 22 | @Override 23 | public String toString() { 24 | return "EnvironmentValue{" + 25 | "key='" + key + '\'' + 26 | ", value='" + value + '\'' + 27 | '}'; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/config/HankSmartClientConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | public interface HankSmartClientConfigurator extends CoordinatorConfigurator { 20 | 21 | public String getRingGroupName(); 22 | 23 | public int getNumConnectionsPerHost(); 24 | 25 | public int getQueryNumMaxTries(); 26 | 27 | public int getTryLockConnectionTimeoutMs(); 28 | 29 | public int getEstablishConnectionTimeoutMs(); 30 | 31 | public int getQueryTimeoutMs(); 32 | 33 | public int getBulkQueryTimeoutMs(); 34 | 35 | public EnvironmentValue getPreferredServerEnvironment(); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/config/SmartClientDaemonConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.config; 17 | 18 | /** 19 | * Configuration particular to the SmartClient Daemon. 20 | */ 21 | public interface SmartClientDaemonConfigurator extends CoordinatorConfigurator { 22 | public String getRingGroupName(); 23 | public int getPortNumber(); 24 | public int getNumThreads(); 25 | } 26 | -------------------------------------------------------------------------------- /hank-client/src/main/java/com/liveramp/hank/config/yaml/YamlClientConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.config.yaml; 17 | 18 | import com.liveramp.hank.config.ClientConfigurator; 19 | import com.liveramp.hank.config.InvalidConfigurationException; 20 | 21 | import java.io.FileNotFoundException; 22 | 23 | public class YamlClientConfigurator extends YamlCoordinatorConfigurator implements ClientConfigurator { 24 | 25 | public YamlClientConfigurator() { 26 | super(); 27 | } 28 | 29 | public YamlClientConfigurator(String configPath) throws FileNotFoundException, InvalidConfigurationException { 30 | super(configPath); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/go/hank/GoUnusedProtection__.go: -------------------------------------------------------------------------------- 1 | // Autogenerated by Thrift Compiler (1.0.0-dev) 2 | // DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 3 | 4 | package hank 5 | 6 | var GoUnusedProtection__ int; 7 | 8 | -------------------------------------------------------------------------------- /hank-core/src/main/go/hank/hank-consts.go: -------------------------------------------------------------------------------- 1 | // Autogenerated by Thrift Compiler (1.0.0-dev) 2 | // DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 3 | 4 | package hank 5 | 6 | import ( 7 | "bytes" 8 | "reflect" 9 | "fmt" 10 | "git.apache.org/thrift.git/lib/go/thrift" 11 | ) 12 | 13 | // (needed to ensure safety because of naive import list construction.) 14 | var _ = thrift.ZERO 15 | var _ = fmt.Printf 16 | var _ = reflect.DeepEqual 17 | var _ = bytes.Equal 18 | 19 | 20 | func init() { 21 | } 22 | 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/CompressionCodec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression; 18 | 19 | import com.liveramp.hank.compression.deflate.DeflateCompressionFactory; 20 | import com.liveramp.hank.compression.none.SlowNoCompressionCompressionFactory; 21 | import com.liveramp.hank.compression.snappy.SnappyCompressionFactory; 22 | import com.liveramp.hank.compression.zip.GzipCompressionFactory; 23 | 24 | public enum CompressionCodec { 25 | DEFLATE, 26 | GZIP, 27 | SNAPPY, 28 | SLOW_NO_COMPRESSION; 29 | 30 | public CompressionFactory getFactory() { 31 | switch (this) { 32 | case DEFLATE: 33 | return new DeflateCompressionFactory(); 34 | case GZIP: 35 | return new GzipCompressionFactory(); 36 | case SNAPPY: 37 | return new SnappyCompressionFactory(); 38 | case SLOW_NO_COMPRESSION: 39 | return new SlowNoCompressionCompressionFactory(); 40 | default: 41 | throw new IllegalStateException(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/CompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression; 18 | 19 | public interface CompressionFactory { 20 | 21 | public Decompressor getDecompressor(); 22 | 23 | public Compressor getCompressor(); 24 | } 25 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/Compressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression; 18 | 19 | import java.io.IOException; 20 | import java.io.OutputStream; 21 | 22 | public interface Compressor { 23 | 24 | public OutputStream getOutputStream(OutputStream outputStream) throws IOException; 25 | } 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/Decompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression; 18 | 19 | import java.io.IOException; 20 | import java.io.OutputStream; 21 | 22 | public interface Decompressor { 23 | 24 | public void decompressBlock(byte[] buffer, int offset, int length, OutputStream outputStream) throws IOException; 25 | } 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/cueball/CueballCompressionCodec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.cueball; 18 | 19 | public interface CueballCompressionCodec { 20 | 21 | public int compress(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOff); 22 | 23 | public int getMaxCompressBufferSize(int length); 24 | 25 | public int decompress(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOff); 26 | } 27 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/cueball/NoCueballCompressionCodec.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.compression.cueball; 2 | 3 | public class NoCueballCompressionCodec implements CueballCompressionCodec { 4 | @Override 5 | public int compress(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOff) { 6 | System.arraycopy(src, srcOffset, dst, dstOff, srcLength); 7 | return srcLength - srcOffset; 8 | } 9 | 10 | @Override 11 | public int decompress(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOff) { 12 | System.arraycopy(src, srcOffset, dst, dstOff, srcLength); 13 | return srcLength - srcOffset; 14 | } 15 | 16 | @Override 17 | public int getMaxCompressBufferSize(int length) { 18 | return length; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/deflate/DeflateCompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.deflate; 18 | 19 | import com.liveramp.hank.compression.CompressionFactory; 20 | import com.liveramp.hank.compression.Compressor; 21 | import com.liveramp.hank.compression.Decompressor; 22 | 23 | public class DeflateCompressionFactory implements CompressionFactory { 24 | 25 | @Override 26 | public Decompressor getDecompressor() { 27 | return new DeflateDecompressor(); 28 | } 29 | 30 | @Override 31 | public Compressor getCompressor() { 32 | return new DeflateCompressor(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/deflate/DeflateCompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.deflate; 18 | 19 | import com.liveramp.hank.compression.Compressor; 20 | 21 | import java.io.OutputStream; 22 | import java.util.zip.Deflater; 23 | import java.util.zip.DeflaterOutputStream; 24 | 25 | public class DeflateCompressor implements Compressor { 26 | 27 | @Override 28 | public OutputStream getOutputStream(OutputStream outputStream) { 29 | Deflater deflater = new Deflater(); 30 | deflater.setLevel(Deflater.BEST_COMPRESSION); 31 | deflater.setStrategy(Deflater.DEFAULT_STRATEGY); 32 | return new DeflaterOutputStream(outputStream, deflater); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/none/SlowNoCompressionCompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.none; 18 | 19 | import com.liveramp.hank.compression.CompressionFactory; 20 | import com.liveramp.hank.compression.Compressor; 21 | import com.liveramp.hank.compression.Decompressor; 22 | 23 | public class SlowNoCompressionCompressionFactory implements CompressionFactory { 24 | 25 | @Override 26 | public Decompressor getDecompressor() { 27 | return new SlowNoCompressionDecompressor(); 28 | } 29 | 30 | @Override 31 | public Compressor getCompressor() { 32 | return new SlowNoCompressionCompressor(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/none/SlowNoCompressionCompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.none; 18 | 19 | import com.liveramp.hank.compression.Compressor; 20 | 21 | import java.io.BufferedOutputStream; 22 | import java.io.OutputStream; 23 | 24 | public class SlowNoCompressionCompressor implements Compressor { 25 | 26 | @Override 27 | public OutputStream getOutputStream(OutputStream outputStream) { 28 | return new BufferedOutputStream(outputStream); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/none/SlowNoCompressionDecompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.none; 18 | 19 | import com.liveramp.hank.compression.common.StreamCopyDecompressor; 20 | import com.liveramp.hank.compression.Decompressor; 21 | 22 | import java.io.BufferedInputStream; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | 26 | public class SlowNoCompressionDecompressor extends StreamCopyDecompressor implements Decompressor { 27 | 28 | @Override 29 | protected InputStream getBlockDecompressionInputStream(InputStream inputStream) throws IOException { 30 | return new BufferedInputStream(inputStream); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/snappy/SnappyCompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.snappy; 18 | 19 | import com.liveramp.hank.compression.CompressionFactory; 20 | import com.liveramp.hank.compression.Compressor; 21 | import com.liveramp.hank.compression.Decompressor; 22 | 23 | public class SnappyCompressionFactory implements CompressionFactory { 24 | 25 | @Override 26 | public Decompressor getDecompressor() { 27 | return new SnappyDecompressor(); 28 | } 29 | 30 | @Override 31 | public Compressor getCompressor() { 32 | return new SnappyCompressor(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/snappy/SnappyCompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.snappy; 18 | 19 | import com.liveramp.hank.compression.Compressor; 20 | import org.xerial.snappy.SnappyOutputStream; 21 | 22 | import java.io.IOException; 23 | import java.io.OutputStream; 24 | 25 | public class SnappyCompressor implements Compressor { 26 | 27 | @Override 28 | public OutputStream getOutputStream(OutputStream outputStream) throws IOException { 29 | return new SnappyOutputStream(outputStream); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/snappy/SnappyDecompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.snappy; 18 | 19 | import com.liveramp.hank.compression.common.StreamCopyDecompressor; 20 | import com.liveramp.hank.compression.Decompressor; 21 | import org.xerial.snappy.SnappyInputStream; 22 | 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | 26 | public class SnappyDecompressor extends StreamCopyDecompressor implements Decompressor { 27 | 28 | @Override 29 | protected InputStream getBlockDecompressionInputStream(InputStream inputStream) throws IOException { 30 | return new SnappyInputStream(inputStream); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/zip/GzipCompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.zip; 18 | 19 | import com.liveramp.hank.compression.CompressionFactory; 20 | import com.liveramp.hank.compression.Compressor; 21 | import com.liveramp.hank.compression.Decompressor; 22 | 23 | public class GzipCompressionFactory implements CompressionFactory { 24 | 25 | @Override 26 | public Decompressor getDecompressor() { 27 | return new GzipDecompressor(); 28 | } 29 | 30 | @Override 31 | public Compressor getCompressor() { 32 | return new GzipCompressor(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/zip/GzipCompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.zip; 18 | 19 | import com.liveramp.hank.compression.Compressor; 20 | 21 | import java.io.IOException; 22 | import java.io.OutputStream; 23 | import java.util.zip.GZIPOutputStream; 24 | 25 | public class GzipCompressor implements Compressor { 26 | 27 | @Override 28 | public OutputStream getOutputStream(OutputStream outputStream) throws IOException { 29 | return new GZIPOutputStream(outputStream); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/compression/zip/GzipDecompressor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.compression.zip; 18 | 19 | import com.liveramp.hank.compression.common.StreamCopyDecompressor; 20 | import com.liveramp.hank.compression.Decompressor; 21 | 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | import java.util.zip.GZIPInputStream; 25 | 26 | public class GzipDecompressor extends StreamCopyDecompressor implements Decompressor { 27 | 28 | @Override 29 | protected InputStream getBlockDecompressionInputStream(InputStream inputStream) throws IOException { 30 | return new GZIPInputStream(inputStream); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/config/CoordinatorConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.config; 17 | 18 | import com.liveramp.hank.coordinator.Coordinator; 19 | 20 | import java.io.Serializable; 21 | 22 | /** 23 | * The base for all configurators that need a Coordinator for all actors in the Hank ecosystem. 24 | */ 25 | public interface CoordinatorConfigurator extends Serializable { 26 | 27 | /** 28 | * Create a new instance of Coordinator for this Hank installation. 29 | */ 30 | public Coordinator createCoordinator(); 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/config/DataDirectoriesConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | import java.util.Set; 20 | 21 | public interface DataDirectoriesConfigurator { 22 | 23 | public Set getDataDirectories(); 24 | } 25 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/config/InvalidConfigurationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.config; 17 | 18 | public class InvalidConfigurationException extends Exception { 19 | public InvalidConfigurationException(String message) { 20 | super(message); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/config/PartitionServerConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | import java.util.Map; 20 | 21 | public interface PartitionServerConfigurator extends CoordinatorConfigurator, ReaderConfigurator { 22 | 23 | public int getServicePort(); 24 | 25 | public String getRingGroupName(); 26 | 27 | public int getNumConcurrentUpdates(); 28 | 29 | public int getMaxConcurrentUpdatesPerDataDirectory(); 30 | 31 | public int getNumConcurrentQueries(); 32 | 33 | public int getNumConcurrentGetBulkTasks(); 34 | 35 | public int getGetBulkTaskSize(); 36 | 37 | public int getGetTimerAggregatorWindow(); 38 | 39 | public long getUpdateFailureCooldown(); 40 | 41 | public Map getEnvironmentFlags(); 42 | 43 | public ReaderConfigurator getReaderConfigurator(int numTotalPartitions); 44 | } 45 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/config/ReaderConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | public interface ReaderConfigurator extends DataDirectoriesConfigurator { 20 | 21 | public long getCacheNumBytesCapacity(); 22 | 23 | public long getCacheNumItemsCapacity(); 24 | 25 | public int getBufferReuseMaxSize(); 26 | } 27 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/AbstractDomainVersion.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | import java.io.IOException; 4 | 5 | public abstract class AbstractDomainVersion implements DomainVersion { 6 | 7 | public int compareTo(DomainVersion domainVersion) { 8 | return Integer.valueOf(getVersionNumber()).compareTo(domainVersion.getVersionNumber()); 9 | } 10 | 11 | @Override 12 | public String toString() { 13 | Boolean isClosed; 14 | Boolean isDefunct; 15 | try { 16 | isClosed = DomainVersions.isClosed(this); 17 | } catch (IOException e) { 18 | isClosed = null; 19 | } 20 | try { 21 | isDefunct = isDefunct(); 22 | } catch (IOException e) { 23 | isDefunct = null; 24 | } 25 | return "AbstractDomainVersion [" 26 | + "version=" + getVersionNumber() 27 | + ", closed=" + isClosed 28 | + ", defunct=" + isDefunct 29 | + "]"; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/AbstractHost.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | import java.io.IOException; 4 | import java.util.SortedSet; 5 | import java.util.TreeSet; 6 | 7 | public abstract class AbstractHost implements Host { 8 | 9 | @Override 10 | public int compareTo(Host o) { 11 | return getAddress().compareTo(o.getAddress()); 12 | } 13 | 14 | @Override 15 | public SortedSet getAssignedDomainsSorted() throws IOException { 16 | return new TreeSet(getAssignedDomains()); 17 | } 18 | 19 | @Override 20 | public HostDomain getHostDomain(Domain domain) { 21 | // TODO: this should be done with a map and caching 22 | try { 23 | for (HostDomain hostDomain : getAssignedDomains()) { 24 | if (hostDomain.getDomain().equals(domain)) { 25 | return hostDomain; 26 | } 27 | } 28 | } catch (IOException e) { 29 | throw new RuntimeException(e); 30 | } 31 | return null; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return String.format("AbstractHost [address=%s]", getAddress()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/AbstractHostDomainPartition.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | 4 | import java.io.IOException; 5 | 6 | public abstract class AbstractHostDomainPartition implements HostDomainPartition { 7 | 8 | @Override 9 | public int compareTo(HostDomainPartition o) { 10 | return Integer.valueOf(getPartitionNumber()).compareTo(o.getPartitionNumber()); 11 | } 12 | 13 | @Override 14 | public String toString() { 15 | try { 16 | return "AbstractHostDomainPartition [partition number=" + getPartitionNumber() + 17 | ", current version=" + getCurrentDomainVersion() + 18 | ", deletable=" + isDeletable() + "]"; 19 | } catch (IOException e) { 20 | throw new RuntimeException(e); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/CloseCoordinatorOpportunistically.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public interface CloseCoordinatorOpportunistically { 20 | 21 | public void closeCoordinatorOpportunistically(Coordinator coordinator); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/CoordinatorFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | import java.util.Map; 19 | 20 | public interface CoordinatorFactory { 21 | 22 | public Coordinator getCoordinator(Map options); 23 | } 24 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/DataLocationChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public interface DataLocationChangeListener { 20 | 21 | public void onDataLocationChange(); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/DomainGroupListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public interface DomainGroupListener { 20 | 21 | public void onDomainGroupChange(DomainGroup domainGroup); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/DomainVersion.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | import com.liveramp.hank.generated.PartitionMetadata; 4 | 5 | import java.io.IOException; 6 | import java.util.Collection; 7 | 8 | public interface DomainVersion extends Comparable { 9 | 10 | public int getVersionNumber(); 11 | 12 | public Long getClosedAt() throws IOException; 13 | 14 | public void close() throws IOException; 15 | 16 | public void cancel() throws IOException; 17 | 18 | public Collection getPartitionsMetadata() throws IOException; 19 | 20 | public void addPartitionProperties(int partNum, long numBytes, long numRecords) throws IOException; 21 | 22 | public boolean isDefunct() throws IOException; 23 | 24 | public void setDefunct(boolean isDefunct) throws IOException; 25 | 26 | public DomainVersionProperties getProperties() throws IOException; 27 | 28 | public void setProperties(DomainVersionProperties properties) throws IOException; 29 | } 30 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/DomainVersionProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public interface DomainVersionProperties { 20 | } 21 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/DomainVersionPropertiesSerialization.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | import java.io.IOException; 20 | 21 | public interface DomainVersionPropertiesSerialization { 22 | 23 | public DomainVersionProperties deserializeProperties(byte[] serializedProperties) throws IOException; 24 | 25 | byte[] serializeProperties(DomainVersionProperties properties) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostCommand.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | public enum HostCommand { 19 | SERVE_DATA, 20 | GO_TO_IDLE, 21 | EXECUTE_UPDATE 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostCommandQueueChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | public interface HostCommandQueueChangeListener { 19 | 20 | public void onCommandQueueChange(Host hostConfig); 21 | } 22 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostDomain.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | import java.io.IOException; 19 | import java.util.Set; 20 | import java.util.SortedSet; 21 | 22 | public interface HostDomain extends Comparable { 23 | 24 | public Domain getDomain(); 25 | 26 | public Set getPartitions() throws IOException; 27 | 28 | public SortedSet getPartitionsSorted() throws IOException; 29 | 30 | public HostDomainPartition getPartitionByNumber(int partNum) throws IOException; 31 | 32 | public HostDomainPartition addPartition(int partNum) throws IOException; 33 | 34 | public void removePartition(int partNum) throws IOException; 35 | } 36 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostDomainPartition.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | import java.io.IOException; 19 | 20 | public interface HostDomainPartition extends Comparable { 21 | 22 | public int getPartitionNumber(); 23 | 24 | public Integer getCurrentDomainVersion() throws IOException; 25 | 26 | public void setCurrentDomainVersion(Integer version) throws IOException; 27 | 28 | public boolean isDeletable() throws IOException; 29 | 30 | public void setDeletable(boolean deletable) throws IOException; 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostDomains.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | import java.io.IOException; 20 | 21 | public final class HostDomains { 22 | 23 | private HostDomains() { 24 | } 25 | 26 | public static void addPartition(HostDomain hostDomain, int partitionNumber) throws IOException { 27 | HostDomainPartition partition = hostDomain.getPartitionByNumber(partitionNumber); 28 | if (partition == null) { 29 | hostDomain.addPartition(partitionNumber); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/HostState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | public enum HostState { 19 | IDLE, 20 | SERVING, 21 | UPDATING, 22 | OFFLINE 23 | } 24 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/PartitionProperties.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | public interface PartitionProperties { 4 | 5 | public int getPartitionNumber(); 6 | 7 | public long getNumBytes(); 8 | 9 | public long getNumRecords(); 10 | } 11 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/Ring.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.coordinator; 17 | 18 | import java.io.IOException; 19 | import java.util.List; 20 | import java.util.Set; 21 | import java.util.SortedSet; 22 | 23 | public interface Ring extends Comparable { 24 | 25 | public RingGroup getRingGroup(); 26 | 27 | public int getRingNumber(); 28 | 29 | public Set getHosts(); 30 | 31 | public SortedSet getHostsSorted(); 32 | 33 | public Host getHostByAddress(PartitionServerAddress address); 34 | 35 | public Host addHost(PartitionServerAddress address, List hostFlags) throws IOException; 36 | 37 | public boolean removeHost(PartitionServerAddress address) throws IOException; 38 | } 39 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/RingGroupDataLocationChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public interface RingGroupDataLocationChangeListener { 20 | 21 | public void onDataLocationChange(RingGroup ringGroup); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/RunnableWithCoordinator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | import java.io.IOException; 20 | 21 | public interface RunnableWithCoordinator { 22 | 23 | public void run(Coordinator coordinator) throws IOException; 24 | } 25 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/ServingStatus.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator; 2 | 3 | public class ServingStatus { 4 | 5 | protected int numPartitions; 6 | protected int numPartitionsServedAndUpToDate; 7 | 8 | public ServingStatus() { 9 | this(0, 0); 10 | } 11 | 12 | public ServingStatus(int numPartitions, int numPartitionsServedAndUpToDate) { 13 | this.numPartitions = numPartitions; 14 | this.numPartitionsServedAndUpToDate = numPartitionsServedAndUpToDate; 15 | } 16 | 17 | public void aggregate(ServingStatus other) { 18 | this.numPartitions += other.numPartitions; 19 | this.numPartitionsServedAndUpToDate += other.numPartitionsServedAndUpToDate; 20 | } 21 | 22 | public int getNumPartitions() { 23 | return numPartitions; 24 | } 25 | 26 | public int getNumPartitionsServedAndUpToDate() { 27 | return numPartitionsServedAndUpToDate; 28 | } 29 | 30 | public void aggregate(int numPartitions, int numPartitionsServedAndUpToDate) { 31 | this.numPartitions += numPartitions; 32 | this.numPartitionsServedAndUpToDate += numPartitionsServedAndUpToDate; 33 | } 34 | 35 | public boolean isServedAndUpToDate() { 36 | return numPartitions == numPartitionsServedAndUpToDate; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/UpdateProgress.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator; 18 | 19 | public class UpdateProgress { 20 | 21 | protected int numPartitions; 22 | protected int numPartitionsUpToDate; 23 | 24 | public UpdateProgress() { 25 | this(0, 0); 26 | } 27 | 28 | public UpdateProgress(int numPartitions, int numPartitionsUpToDate) { 29 | this.numPartitions = numPartitions; 30 | this.numPartitionsUpToDate = numPartitionsUpToDate; 31 | } 32 | 33 | public int getNumPartitions() { 34 | return numPartitions; 35 | } 36 | 37 | public int getNumPartitionsUpToDate() { 38 | return numPartitionsUpToDate; 39 | } 40 | 41 | public float getUpdateProgress() { 42 | if (numPartitions <= 0) { 43 | return 0; 44 | } else { 45 | return (float) numPartitionsUpToDate / (float) numPartitions; 46 | } 47 | } 48 | 49 | public void aggregate(UpdateProgress other) { 50 | numPartitions += other.numPartitions; 51 | numPartitionsUpToDate += other.numPartitionsUpToDate; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/zk/DotComplete.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator.zk; 2 | 3 | import org.apache.zookeeper.KeeperException; 4 | import org.apache.zookeeper.WatchedEvent; 5 | import org.apache.zookeeper.Watcher; 6 | import org.apache.zookeeper.Watcher.Event.KeeperState; 7 | 8 | import com.liveramp.hank.zookeeper.ZkPath; 9 | import com.liveramp.hank.zookeeper.ZooKeeperPlus; 10 | import com.liveramp.hank.zookeeper.WatchedMap.CompletionAwaiter; 11 | import com.liveramp.hank.zookeeper.WatchedMap.CompletionDetector; 12 | 13 | public class DotComplete implements CompletionDetector { 14 | 15 | public static final String NODE_NAME = ".complete"; 16 | 17 | public static final class CreationWatcher implements Watcher { 18 | private final String relPath; 19 | private final CompletionAwaiter awaiter; 20 | 21 | public CreationWatcher(String relPath, CompletionAwaiter awaiter) { 22 | this.relPath = relPath; 23 | this.awaiter = awaiter; 24 | } 25 | 26 | @Override 27 | public void process(WatchedEvent event) { 28 | if (event.getState() != KeeperState.SyncConnected) { 29 | return; 30 | } 31 | switch (event.getType()) { 32 | case NodeCreated: 33 | awaiter.completed(relPath); 34 | } 35 | } 36 | } 37 | 38 | @Override 39 | public void detectCompletion(ZooKeeperPlus zk, String basePath, String relPath, CompletionAwaiter awaiter) throws KeeperException, InterruptedException { 40 | if (zk.exists(ZkPath.append(basePath, relPath, NODE_NAME), new CreationWatcher(relPath, awaiter)) != null) { 41 | awaiter.completed(relPath); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/coordinator/zk/MigrationHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.coordinator.zk; 18 | 19 | import com.liveramp.hank.config.InvalidConfigurationException; 20 | import com.liveramp.hank.config.yaml.YamlCoordinatorConfigurator; 21 | import com.liveramp.hank.coordinator.Coordinator; 22 | import com.liveramp.hank.util.CommandLineChecker; 23 | import org.slf4j.Logger; import org.slf4j.LoggerFactory; 24 | import org.apache.zookeeper.KeeperException; 25 | 26 | import java.io.IOException; 27 | 28 | public class MigrationHelper { 29 | 30 | private static Logger LOG = LoggerFactory.getLogger(MigrationHelper.class); 31 | 32 | public static void main(String[] args) throws IOException, InvalidConfigurationException, InterruptedException, KeeperException { 33 | CommandLineChecker.check(args, new String[]{"configuration"}, 34 | MigrationHelper.class); 35 | 36 | String configurationPath = args[0]; 37 | 38 | Coordinator coordinator = new YamlCoordinatorConfigurator(configurationPath).createCoordinator(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/hasher/Hasher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.hasher; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | /** 21 | * Interface for general-purpose hashing functions. 22 | */ 23 | public interface Hasher { 24 | public void hash(ByteBuffer value, int hashSize, byte[] hashBytes); 25 | } 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/hasher/IdentityHasher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hasher; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | public class IdentityHasher implements Hasher { 22 | 23 | @Override 24 | public void hash(ByteBuffer value, int hashSize, byte[] hashBytes) { 25 | if (value.remaining() != hashSize) { 26 | throw new IllegalStateException("Tried to compute an identity hash of " + hashSize + " bytes with an incoming value of " + value.remaining() + " bytes."); 27 | } 28 | System.arraycopy(value.array(), value.arrayOffset() + value.position(), hashBytes, 0, hashSize); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/hasher/LeftPaddedIdentityHasher.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.hasher; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | public class LeftPaddedIdentityHasher implements Hasher { 6 | @Override 7 | public void hash(ByteBuffer value, int hashSize, byte[] hashBytes) { 8 | 9 | if (value.remaining() > hashSize) { 10 | throw new IllegalArgumentException("Cannot pad incoming item "+value+" to length "+hashSize+"!"); 11 | } 12 | 13 | int bytesToPad = hashSize - value.remaining(); 14 | 15 | System.arraycopy(value.array(), value.arrayOffset() + value.position(), hashBytes, bytesToPad, value.remaining()); 16 | 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partition_assigner/PartitionAssigner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.partition_assigner; 18 | 19 | import java.io.IOException; 20 | import java.util.Set; 21 | 22 | import com.liveramp.hank.coordinator.DomainAndVersion; 23 | import com.liveramp.hank.coordinator.Host; 24 | import com.liveramp.hank.coordinator.Ring; 25 | import com.liveramp.hank.ring_group_conductor.RingGroupConductorMode; 26 | 27 | public interface PartitionAssigner { 28 | 29 | public void prepare(Ring ring, 30 | Set domainVersions, 31 | RingGroupConductorMode ringGroupConductorMode) throws IOException; 32 | 33 | public boolean isAssigned(Host host) throws IOException; 34 | 35 | public void assign(Host host) throws IOException; 36 | } 37 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partition_server/DiskPartitionAssignment.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.partition_server; 2 | 3 | import java.util.Map; 4 | 5 | public class DiskPartitionAssignment { 6 | 7 | private final Map dataDirectoryPerPartition; 8 | 9 | public DiskPartitionAssignment(Map dataDirectoryPerPartition) { 10 | this.dataDirectoryPerPartition = dataDirectoryPerPartition; 11 | } 12 | 13 | public String getDisk(int partitionNumber) { 14 | return dataDirectoryPerPartition.get(partitionNumber); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partition_server/IfaceWithShutdown.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.partition_server; 2 | 3 | import com.liveramp.hank.generated.PartitionServer.Iface; 4 | 5 | public interface IfaceWithShutdown extends Iface { 6 | 7 | public void shutDown() throws InterruptedException; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partition_server/PartitionUpdateTaskStatistics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.partition_server; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | public class PartitionUpdateTaskStatistics { 23 | 24 | private long startTimeMs; 25 | private long endTimeMs; 26 | private final Map durationsMs; 27 | 28 | public PartitionUpdateTaskStatistics() { 29 | this.durationsMs = new HashMap(); 30 | } 31 | 32 | public long getStartTimeMs() { 33 | return startTimeMs; 34 | } 35 | 36 | void setStartTimeMs(long startTimeMs) { 37 | this.startTimeMs = startTimeMs; 38 | } 39 | 40 | public long getEndTimeMs() { 41 | return endTimeMs; 42 | } 43 | 44 | void setEndTimeMs(long endTimeMs) { 45 | this.endTimeMs = endTimeMs; 46 | } 47 | 48 | public Map getDurationsMs() { 49 | return durationsMs; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partitioner/ConstantPartitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * 18 | */ 19 | package com.liveramp.hank.partitioner; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public final class ConstantPartitioner implements Partitioner { 24 | @Override 25 | public int partition(ByteBuffer key, int numPartitions) { 26 | return 0; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partitioner/HashCodePartitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.partitioner; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | public class HashCodePartitioner implements Partitioner { 22 | 23 | @Override 24 | public int partition(ByteBuffer key, int numPartitions) { 25 | int hashCode = 31 + (key != null ? bytesHashCode(key.array(), key.arrayOffset() + key.position(), key.remaining()) : 0); 26 | return (hashCode & Integer.MAX_VALUE) % numPartitions; 27 | } 28 | 29 | private int bytesHashCode(byte[] bytes, int offset, int length) { 30 | int hash = 1; 31 | for (int i = offset; i < offset + length; i++) { 32 | hash = (31 * hash) + (int)bytes[i]; 33 | } 34 | return hash; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/partitioner/Partitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.partitioner; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | /** 21 | * Interface used to partition keys across a domain's partitions. 22 | */ 23 | public interface Partitioner { 24 | /** 25 | * The returned partition number must be geq 0 and deterministic 26 | * @param key 27 | * @return 28 | */ 29 | public int partition(ByteBuffer key, int numPartitions); 30 | } 31 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/ring_group_conductor/RingGroupConductorMode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.ring_group_conductor; 18 | 19 | public enum RingGroupConductorMode { 20 | OFFLINE, 21 | INACTIVE, 22 | ACTIVE, 23 | PROACTIVE, // TODO pretty sure we don't want this 24 | AUTOCONFIGURE 25 | } 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/Compactor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage; 18 | 19 | import com.liveramp.hank.coordinator.CloseCoordinatorOpportunistically; 20 | import com.liveramp.hank.coordinator.DomainVersion; 21 | 22 | import java.io.IOException; 23 | 24 | public interface Compactor extends CloseCoordinatorOpportunistically { 25 | 26 | public void compact(DomainVersion versionToCompact, 27 | Writer writer) throws IOException; 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/Deleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * Interface through which individual partitions are deleted. 7 | */ 8 | public interface Deleter { 9 | public void delete() throws IOException; 10 | } 11 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/FileOpsUtil.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage; 2 | 3 | import java.util.Map; 4 | 5 | import static com.liveramp.hank.storage.curly.Curly.Factory.REMOTE_DOMAIN_ROOT_KEY; 6 | 7 | public class FileOpsUtil { 8 | 9 | public static final String PARTITION_SERVER_DOMAIN_ROOT_KEY = "partition_server_remote_domain_root"; 10 | public static final String DOMAIN_BUILDER_DOMAIN_ROOT_KEY = "domain_builder_remote_domain_root"; 11 | 12 | 13 | public static String getPartitionServerRoot(Map options){ 14 | 15 | if(options.containsKey(REMOTE_DOMAIN_ROOT_KEY)){ 16 | return (String)options.get(REMOTE_DOMAIN_ROOT_KEY); 17 | } 18 | 19 | return (String) options.get(PARTITION_SERVER_DOMAIN_ROOT_KEY); 20 | 21 | } 22 | 23 | public static String getDomainBuilderRoot(Map options){ 24 | 25 | if(options.containsKey(REMOTE_DOMAIN_ROOT_KEY)){ 26 | return (String)options.get(REMOTE_DOMAIN_ROOT_KEY); 27 | } 28 | 29 | return (String) options.get(DOMAIN_BUILDER_DOMAIN_ROOT_KEY); 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/PartitionRemoteFileOps.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.io.OutputStream; 22 | 23 | public interface PartitionRemoteFileOps { 24 | 25 | public InputStream getInputStream(String remoteRelativePath) throws IOException; 26 | 27 | public OutputStream getOutputStream(String remoteRelativePath) throws IOException; 28 | 29 | public boolean exists(String remoteRelativePath) throws IOException; 30 | 31 | public void copyToLocalRoot(String remoteSourceRelativePath, String localDestinationRoot) throws IOException; 32 | 33 | public boolean attemptDelete(String remoteRelativePath) throws IOException; 34 | 35 | public String getRemoteAbsolutePath(String remoteRelativePath); 36 | } 37 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/PartitionRemoteFileOpsFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage; 17 | 18 | import java.io.IOException; 19 | 20 | /** 21 | * Factory for getting PartitionRemoteFileOps instances. 22 | */ 23 | public interface PartitionRemoteFileOpsFactory { 24 | 25 | public PartitionRemoteFileOps getPartitionRemoteFileOps(String remoteRoot, int partitionNumber) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/PartitionUpdater.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage; 18 | 19 | import com.liveramp.hank.coordinator.DomainVersion; 20 | import com.liveramp.hank.partition_server.PartitionUpdateTaskStatistics; 21 | 22 | import java.io.IOException; 23 | 24 | public interface PartitionUpdater { 25 | 26 | void updateTo(DomainVersion updatingToVersion, PartitionUpdateTaskStatistics statistics) throws IOException; 27 | } 28 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/Reader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage; 17 | 18 | import java.io.IOException; 19 | import java.nio.ByteBuffer; 20 | 21 | /** 22 | * Interface through which individual partitions are queried. 23 | */ 24 | public interface Reader { 25 | 26 | public void get(ByteBuffer key, ReaderResult result) throws IOException; 27 | 28 | // null means no versioning 29 | public Integer getVersionNumber(); 30 | 31 | public CacheStatistics getCacheStatistics(); 32 | 33 | public void close() throws IOException; 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/RemoteDomainCleaner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage; 18 | 19 | import java.io.IOException; 20 | 21 | public interface RemoteDomainCleaner { 22 | public void deleteOldVersions(RemoteDomainVersionDeleter remoteDomainVersionDeleter, boolean deleteMetadata) throws IOException; 23 | } 24 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/RemoteDomainVersionDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage; 2 | 3 | import java.io.IOException; 4 | 5 | public interface RemoteDomainVersionDeleter { 6 | /** 7 | * Purge the specified version from remote storage. 8 | * 9 | * @param versionNumber 10 | * @throws IOException 11 | */ 12 | public void deleteVersion(int versionNumber) throws IOException; 13 | } 14 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/StorageEngineFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage; 17 | 18 | import com.liveramp.hank.coordinator.Domain; 19 | 20 | import java.io.IOException; 21 | import java.util.Map; 22 | 23 | public interface StorageEngineFactory { 24 | 25 | public String getPrettyName(); 26 | 27 | public String getDefaultOptions(); 28 | 29 | public StorageEngine getStorageEngine(Map options, Domain domain) throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/Writer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage; 17 | 18 | import java.io.IOException; 19 | import java.nio.ByteBuffer; 20 | 21 | public interface Writer { 22 | public void write(ByteBuffer key, ByteBuffer value) throws IOException; 23 | 24 | void close() throws IOException; 25 | 26 | public long getNumBytesWritten(); 27 | 28 | public long getNumRecordsWritten(); 29 | } 30 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/CueballDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.cueball; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | import com.liveramp.hank.storage.Deleter; 7 | import org.apache.commons.io.FileUtils; 8 | 9 | public class CueballDeleter implements Deleter { 10 | private final String localPartitionRoot; 11 | 12 | public CueballDeleter(String localPartitionRoot) { 13 | this.localPartitionRoot = localPartitionRoot; 14 | } 15 | 16 | @Override 17 | public void delete() throws IOException { 18 | FileUtils.deleteDirectory(new File(localPartitionRoot)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/CueballFilePath.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.cueball; 18 | 19 | import com.liveramp.hank.storage.PartitionFileLocalPath; 20 | 21 | import java.io.File; 22 | 23 | public class CueballFilePath extends PartitionFileLocalPath { 24 | 25 | public CueballFilePath(String path) { 26 | super(path, Cueball.parseVersionNumber(new File(path).getName())); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/CueballRemoteDomainCleaner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.cueball; 18 | 19 | import com.liveramp.hank.coordinator.Domain; 20 | import com.liveramp.hank.coordinator.DomainVersion; 21 | import com.liveramp.hank.storage.incremental.IncrementalDomainVersionProperties; 22 | import com.liveramp.hank.storage.incremental.IncrementalRemoteDomainCleaner; 23 | import com.liveramp.hank.storage.RemoteDomainCleaner; 24 | 25 | import java.io.IOException; 26 | 27 | public class CueballRemoteDomainCleaner extends IncrementalRemoteDomainCleaner implements RemoteDomainCleaner { 28 | 29 | public CueballRemoteDomainCleaner(Domain domain, 30 | int numRemoteLeafVersionsToKeep) { 31 | super(domain, numRemoteLeafVersionsToKeep); 32 | } 33 | 34 | protected DomainVersion getParentDomainVersion(Domain domain, DomainVersion domainVersion) throws IOException { 35 | return IncrementalDomainVersionProperties.getParentDomainVersion(domain, domainVersion); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/CueballRemoteDomainVersionDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.cueball; 2 | 3 | import com.liveramp.hank.coordinator.Domain; 4 | import com.liveramp.hank.storage.PartitionRemoteFileOps; 5 | import com.liveramp.hank.storage.PartitionRemoteFileOpsFactory; 6 | import com.liveramp.hank.storage.RemoteDomainVersionDeleter; 7 | 8 | import java.io.IOException; 9 | 10 | public class CueballRemoteDomainVersionDeleter implements RemoteDomainVersionDeleter { 11 | 12 | protected final Domain domain; 13 | protected final String remoteDomainRoot; 14 | protected final PartitionRemoteFileOpsFactory fileOpsFactory; 15 | 16 | public CueballRemoteDomainVersionDeleter(Domain domain, 17 | String remoteDomainRoot, 18 | PartitionRemoteFileOpsFactory fileOpsFactory) { 19 | this.domain = domain; 20 | this.remoteDomainRoot = remoteDomainRoot; 21 | this.fileOpsFactory = fileOpsFactory; 22 | } 23 | 24 | @Override 25 | public void deleteVersion(int versionNumber) throws IOException { 26 | for (int partition = 0; partition < domain.getNumParts(); ++partition) { 27 | PartitionRemoteFileOps fileOps = fileOpsFactory.getPartitionRemoteFileOps(remoteDomainRoot, partition); 28 | fileOps.attemptDelete(Cueball.getName(versionNumber, true)); 29 | fileOps.attemptDelete(Cueball.getName(versionNumber, false)); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/HashPrefixCalculator.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.cueball; 2 | 3 | public final class HashPrefixCalculator { 4 | private final int numBits; 5 | 6 | public HashPrefixCalculator(int numBits) { 7 | this.numBits = numBits; 8 | } 9 | 10 | public final int getHashPrefix(final byte[] chunkBytes, int off) { 11 | final int bitsFromLastByte = numBits % 8; 12 | final int numFullBytes = off + (numBits / 8); 13 | 14 | int prefix = 0; 15 | for (; off < numFullBytes; off++) { 16 | prefix = (prefix << 8) | (chunkBytes[off] & 0xff); 17 | } 18 | if (bitsFromLastByte == 0) { 19 | return prefix; 20 | } else { 21 | return (prefix << bitsFromLastByte) | (((chunkBytes[off] & 0xff) >> (8 - bitsFromLastByte))); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/ICueballMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage.cueball; 17 | 18 | import com.liveramp.hank.compression.cueball.CueballCompressionCodec; 19 | 20 | import java.io.IOException; 21 | import java.util.List; 22 | 23 | public interface ICueballMerger { 24 | 25 | public void merge(final CueballFilePath latestBase, 26 | final List deltas, 27 | final String newBasePath, 28 | final int keyHashSize, 29 | final int valueSize, 30 | ValueTransformer transformer, 31 | int hashIndexBits, 32 | CueballCompressionCodec compressionCodec) 33 | throws IOException; 34 | } 35 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/ICueballStreamBufferMergeSortFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.cueball; 18 | 19 | import java.io.IOException; 20 | import java.util.List; 21 | 22 | public interface ICueballStreamBufferMergeSortFactory { 23 | 24 | public IKeyFileStreamBufferMergeSort getInstance(CueballFilePath cueballBase, 25 | List cueballDeltas) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/IKeyFileStreamBufferMergeSort.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.cueball; 18 | 19 | import java.io.IOException; 20 | 21 | public interface IKeyFileStreamBufferMergeSort { 22 | 23 | public KeyHashAndValueAndStreamIndex nextKeyHashAndValueAndStreamIndex() throws IOException; 24 | 25 | public void close() throws IOException; 26 | 27 | public int getNumStreams(); 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/KeyHashAndValueAndStreamIndex.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.cueball; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | public class KeyHashAndValueAndStreamIndex { 22 | public final ByteBuffer keyHash; 23 | public final ByteBuffer value; 24 | public final int streamIndex; 25 | 26 | public KeyHashAndValueAndStreamIndex(ByteBuffer keyHash, ByteBuffer value, int streamIndex) { 27 | this.keyHash = keyHash; 28 | this.value = value; 29 | this.streamIndex = streamIndex; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/cueball/ValueTransformer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.storage.cueball; 17 | 18 | 19 | public interface ValueTransformer { 20 | public void transform(byte[] buf, int off, int relIndex); 21 | } 22 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/CurlyDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.curly; 2 | 3 | import com.liveramp.hank.storage.Deleter; 4 | import org.apache.commons.io.FileUtils; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | public class CurlyDeleter implements Deleter { 10 | private final String localPartitionRoot; 11 | 12 | public CurlyDeleter(String localPartitionRoot) { 13 | this.localPartitionRoot = localPartitionRoot; 14 | } 15 | 16 | @Override 17 | public void delete() throws IOException { 18 | FileUtils.deleteDirectory(new File(localPartitionRoot)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/CurlyFilePath.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.storage.PartitionFileLocalPath; 20 | 21 | import java.io.File; 22 | 23 | public class CurlyFilePath extends PartitionFileLocalPath { 24 | 25 | public CurlyFilePath(String path) { 26 | super(path, Curly.parseVersionNumber(new File(path).getName())); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/CurlyRemoteDomainCleaner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.coordinator.Domain; 20 | import com.liveramp.hank.coordinator.DomainVersion; 21 | import com.liveramp.hank.storage.incremental.IncrementalDomainVersionProperties; 22 | import com.liveramp.hank.storage.incremental.IncrementalRemoteDomainCleaner; 23 | import com.liveramp.hank.storage.RemoteDomainCleaner; 24 | 25 | import java.io.IOException; 26 | 27 | public class CurlyRemoteDomainCleaner extends IncrementalRemoteDomainCleaner implements RemoteDomainCleaner { 28 | 29 | public CurlyRemoteDomainCleaner(Domain domain, 30 | int numRemoteLeafVersionsToKeep) { 31 | super(domain, numRemoteLeafVersionsToKeep); 32 | } 33 | 34 | protected DomainVersion getParentDomainVersion(Domain domain, DomainVersion domainVersion) throws IOException { 35 | return IncrementalDomainVersionProperties.getParentDomainVersion(domain, domainVersion); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/CurlyRemoteDomainVersionDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.curly; 2 | 3 | import com.liveramp.hank.coordinator.Domain; 4 | import com.liveramp.hank.storage.PartitionRemoteFileOps; 5 | import com.liveramp.hank.storage.PartitionRemoteFileOpsFactory; 6 | import com.liveramp.hank.storage.RemoteDomainVersionDeleter; 7 | import com.liveramp.hank.storage.cueball.Cueball; 8 | 9 | import java.io.IOException; 10 | 11 | public class CurlyRemoteDomainVersionDeleter implements RemoteDomainVersionDeleter { 12 | 13 | protected final Domain domain; 14 | protected final String remoteDomainRoot; 15 | protected final PartitionRemoteFileOpsFactory fileOpsFactory; 16 | 17 | public CurlyRemoteDomainVersionDeleter(Domain domain, 18 | String remoteDomainRoot, 19 | PartitionRemoteFileOpsFactory fileOpsFactory) { 20 | this.domain = domain; 21 | this.remoteDomainRoot = remoteDomainRoot; 22 | this.fileOpsFactory = fileOpsFactory; 23 | } 24 | 25 | @Override 26 | public void deleteVersion(int versionNumber) throws IOException { 27 | for (int partition = 0; partition < domain.getNumParts(); ++partition) { 28 | PartitionRemoteFileOps fileOps = fileOpsFactory.getPartitionRemoteFileOps(remoteDomainRoot, partition); 29 | fileOps.attemptDelete(Cueball.getName(versionNumber, true)); 30 | fileOps.attemptDelete(Cueball.getName(versionNumber, false)); 31 | 32 | fileOps.attemptDelete(Curly.getName(versionNumber, true)); 33 | fileOps.attemptDelete(Curly.getName(versionNumber, false)); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/ICurlyCompactingMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.storage.Writer; 20 | import com.liveramp.hank.storage.cueball.IKeyFileStreamBufferMergeSort; 21 | 22 | import java.io.IOException; 23 | import java.util.List; 24 | 25 | public interface ICurlyCompactingMerger { 26 | 27 | public void merge(final CurlyFilePath curlyBasePath, 28 | final List curlyDeltas, 29 | final IKeyFileStreamBufferMergeSort keyFileStreamBufferMergeSort, 30 | final ICurlyReaderFactory curlyReaderFactory, 31 | final Writer recordFileWriter) throws IOException; 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/ICurlyMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.storage.PartitionRemoteFileOps; 20 | 21 | import java.io.IOException; 22 | import java.util.List; 23 | 24 | public interface ICurlyMerger { 25 | 26 | public long[] merge(final CurlyFilePath latestBase, 27 | final List deltaRemoteFiles, 28 | final PartitionRemoteFileOps partitionRemoteFileOps) 29 | throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/ICurlyReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.storage.Reader; 20 | import com.liveramp.hank.storage.ReaderResult; 21 | 22 | import java.io.IOException; 23 | import java.nio.ByteBuffer; 24 | 25 | public interface ICurlyReader extends Reader { 26 | 27 | public void readRecord(ByteBuffer location, ReaderResult result) throws IOException; 28 | } 29 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/curly/ICurlyReaderFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import java.io.IOException; 20 | 21 | public interface ICurlyReaderFactory { 22 | 23 | public ICurlyReader getInstance(CurlyFilePath curlyFilePath) throws IOException; 24 | } 25 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/echo/EchoDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.echo; 2 | 3 | import com.liveramp.hank.storage.Deleter; 4 | 5 | public class EchoDeleter implements Deleter { 6 | private final int partNum; 7 | 8 | public EchoDeleter(int partNum) { 9 | this.partNum = partNum; 10 | } 11 | 12 | public void delete() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/echo/EchoReader.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.echo; 2 | 3 | import com.liveramp.commons.util.BytesUtils; 4 | import com.liveramp.hank.storage.CacheStatistics; 5 | import com.liveramp.hank.storage.Reader; 6 | import com.liveramp.hank.storage.ReaderResult; 7 | 8 | import java.io.IOException; 9 | import java.nio.ByteBuffer; 10 | 11 | public class EchoReader implements Reader { 12 | private final int partNum; 13 | 14 | public EchoReader(int partNum) { 15 | this.partNum = partNum; 16 | } 17 | 18 | @Override 19 | public void get(ByteBuffer key, ReaderResult result) throws IOException { 20 | StringBuilder sb = new StringBuilder(); 21 | sb.append("Original value: "); 22 | sb.append(BytesUtils.bytesToHexString(key)); 23 | sb.append(" Assigned to partition number: "); 24 | sb.append(partNum); 25 | 26 | byte[] bytes = sb.toString().getBytes(); 27 | result.requiresBufferSize(bytes.length); 28 | System.arraycopy(bytes, 0, result.getBuffer().array(), 0, bytes.length); 29 | result.found(); 30 | } 31 | 32 | @Override 33 | public Integer getVersionNumber() { 34 | return null; 35 | } 36 | 37 | @Override 38 | public CacheStatistics getCacheStatistics() { 39 | return null; 40 | } 41 | 42 | @Override 43 | public void close() { 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/echo/EchoUpdater.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.echo; 2 | 3 | import com.liveramp.hank.coordinator.DomainVersion; 4 | import com.liveramp.hank.partition_server.PartitionUpdateTaskStatistics; 5 | import com.liveramp.hank.storage.PartitionUpdater; 6 | 7 | import java.io.IOException; 8 | 9 | public class EchoUpdater implements PartitionUpdater { 10 | 11 | @Override 12 | public void updateTo(DomainVersion updatingToVersion, PartitionUpdateTaskStatistics statistics) throws IOException { 13 | // No-op 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/storage/incremental/IncrementalStorageEngine.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.incremental; 18 | 19 | import com.liveramp.hank.coordinator.Domain; 20 | import com.liveramp.hank.storage.StorageEngine; 21 | 22 | public abstract class IncrementalStorageEngine implements StorageEngine { 23 | 24 | public abstract IncrementalUpdatePlanner getUpdatePlanner(Domain domain); 25 | } 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/test/hasher/MapHasher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.test.hasher; 17 | 18 | import com.liveramp.hank.hasher.Hasher; 19 | 20 | import java.nio.ByteBuffer; 21 | import java.util.Map; 22 | 23 | 24 | public class MapHasher implements Hasher { 25 | private final Map staticHashes; 26 | 27 | public MapHasher(Map staticHashes) { 28 | this.staticHashes = staticHashes; 29 | } 30 | 31 | @Override 32 | public void hash(ByteBuffer value, int hashSize, byte[] hashBytes) { 33 | byte[] hsh = staticHashes.get(value); 34 | if (hsh == null) { 35 | throw new RuntimeException("No hash set for key!"); 36 | } 37 | if (hsh.length != hashSize) { 38 | throw new RuntimeException("Incompatible hash size: " + hsh.length + " and " + hashSize); 39 | } 40 | System.arraycopy(hsh, 0, hashBytes, 0, hashSize); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/CommandLineChecker.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.util; 2 | 3 | import org.apache.commons.cli.CommandLine; 4 | import org.apache.commons.cli.HelpFormatter; 5 | import org.apache.commons.cli.Options; 6 | 7 | public final class CommandLineChecker { 8 | 9 | public static final void check(CommandLine commandLine, Options options, String[] requiredOptions, Class clazz) { 10 | for (String option : requiredOptions) { 11 | if (!commandLine.hasOption(option)) { 12 | HelpFormatter formatter = new HelpFormatter(); 13 | formatter.printHelp(clazz.getSimpleName(), options); 14 | System.exit(1); 15 | } 16 | } 17 | } 18 | 19 | public static final void check(String[] arguments, String[] expectedArguments, Class clazz) { 20 | if (arguments.length != expectedArguments.length) { 21 | StringBuilder usage = new StringBuilder(); 22 | usage.append(clazz.getSimpleName()); 23 | for (String arg : expectedArguments) { 24 | usage.append(" <"); 25 | usage.append(arg); 26 | usage.append(">"); 27 | } 28 | System.err.println(usage); 29 | System.exit(1); 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/HankResponseMemoryUsageEstimator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.util; 18 | 19 | import java.io.Serializable; 20 | 21 | import com.liveramp.commons.util.MemoryUsageEstimator; 22 | import com.liveramp.hank.generated.HankResponse; 23 | 24 | public class HankResponseMemoryUsageEstimator implements MemoryUsageEstimator, Serializable { 25 | @Override 26 | public long estimateMemorySize(HankResponse item) { 27 | if (item.is_set_value()) { 28 | return item.get_value().length; 29 | } else { 30 | return 1; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/HankTimer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.util; 18 | 19 | public class HankTimer { 20 | 21 | private long startTime; 22 | 23 | public HankTimer() { 24 | startTime = System.nanoTime(); 25 | } 26 | 27 | public void restart() { 28 | startTime = System.nanoTime(); 29 | } 30 | 31 | public long getStartTime() { 32 | return startTime; 33 | } 34 | 35 | public long getDuration() { 36 | return Math.abs(startTime - System.nanoTime()); // Since nanoTime can return negative values 37 | } 38 | 39 | public long getDurationMs() { 40 | return getDuration() / 1000000; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/IOStreamUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.util; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.io.OutputStream; 22 | 23 | public class IOStreamUtils { 24 | 25 | public static final int DEFAULT_BUFFER_SIZE = 256 * 1024; 26 | 27 | public static long copy(final InputStream input, 28 | final OutputStream output) throws IOException { 29 | byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; 30 | return copy(input, output, buffer); 31 | } 32 | 33 | public static long copy(final InputStream input, 34 | final OutputStream output, 35 | final byte[] buffer) throws IOException { 36 | long count = 0; 37 | int n = 0; 38 | while (-1 != (n = input.read(buffer))) { 39 | output.write(buffer, 0, n); 40 | count += n; 41 | } 42 | return count; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/LocalHostUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.util; 17 | 18 | import java.net.InetAddress; 19 | import java.net.UnknownHostException; 20 | 21 | public final class LocalHostUtils { 22 | private LocalHostUtils() {} 23 | 24 | public static String getHostName() throws UnknownHostException { 25 | return InetAddress.getLocalHost().getHostName(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/util/ReverseComparator.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.util; 2 | 3 | import java.util.Comparator; 4 | 5 | public class ReverseComparator> implements Comparator { 6 | @Override 7 | public int compare(T arg0, T arg1) { 8 | return -1 * arg0.compareTo(arg1); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedBoolean.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.zookeeper; 2 | 3 | import org.apache.zookeeper.KeeperException; 4 | 5 | public class WatchedBoolean extends WatchedNode { 6 | 7 | public WatchedBoolean(ZooKeeperPlus zk, String nodePath, boolean waitForCreation) 8 | throws KeeperException, InterruptedException { 9 | super(zk, nodePath, waitForCreation); 10 | } 11 | 12 | public static Boolean get(ZooKeeperPlus zk, String nodePath) throws InterruptedException, KeeperException { 13 | return decodeValue(zk.getData(nodePath, null, null)); 14 | } 15 | 16 | protected static Boolean decodeValue(byte[] data) { 17 | if (data == null) { 18 | return null; 19 | } 20 | return Boolean.parseBoolean(new String(data)); 21 | } 22 | 23 | protected static byte[] encodeValue(Boolean v) { 24 | if (v == null) { 25 | return null; 26 | } 27 | return v.toString().getBytes(); 28 | } 29 | 30 | @Override 31 | protected Boolean decode(byte[] data) { 32 | return decodeValue(data); 33 | } 34 | 35 | @Override 36 | protected byte[] encode(Boolean v) { 37 | return encodeValue(v); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedBytes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | import org.apache.zookeeper.KeeperException; 20 | 21 | public class WatchedBytes extends WatchedNode { 22 | 23 | /** 24 | * Start watching a node, optionnaly waiting for it to be created 25 | * 26 | * @param zk 27 | * @param nodePath 28 | * @param waitForCreation 29 | * @throws org.apache.zookeeper.KeeperException 30 | * 31 | * @throws InterruptedException 32 | */ 33 | public WatchedBytes(final ZooKeeperPlus zk, 34 | final String nodePath, 35 | boolean waitForCreation) throws KeeperException, InterruptedException { 36 | super(zk, nodePath, waitForCreation); 37 | } 38 | 39 | @Override 40 | protected byte[] decode(byte[] data) { 41 | return data; 42 | } 43 | 44 | @Override 45 | protected byte[] encode(byte[] v) { 46 | return v; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedLong.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.zookeeper; 2 | 3 | import org.apache.zookeeper.KeeperException; 4 | 5 | public class WatchedLong extends WatchedNode { 6 | 7 | public WatchedLong(ZooKeeperPlus zk, String nodePath, boolean waitForCreation) 8 | throws KeeperException, InterruptedException { 9 | super(zk, nodePath, waitForCreation); 10 | } 11 | 12 | public WatchedLong(ZooKeeperPlus zk, String nodePath) 13 | throws KeeperException, InterruptedException { 14 | this(zk, nodePath, false); 15 | } 16 | 17 | @Override 18 | protected Long decode(byte[] data) { 19 | if (data == null) { 20 | return null; 21 | } 22 | return Long.parseLong(new String(data)); 23 | } 24 | 25 | @Override 26 | protected byte[] encode(Long v) { 27 | if (v == null) { 28 | return null; 29 | } 30 | return v.toString().getBytes(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedMapListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | public interface WatchedMapListener { 20 | 21 | public void onWatchedMapChange(WatchedMap watchedMap); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedNodeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | public interface WatchedNodeListener { 20 | 21 | public abstract void onWatchedNodeChange(T value); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedNodeUpdater.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | public interface WatchedNodeUpdater { 20 | 21 | // Note: update() should not modify its argument 22 | public T update(T current); 23 | } 24 | 25 | 26 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedNodeUpdaterWithReturnValue.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | interface WatchedNodeUpdaterWithReturnValue extends WatchedNodeUpdater { 20 | 21 | public R getReturnValue(); 22 | } 23 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedNodeUpdaterWithReturnValueImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.zookeeper; 18 | 19 | public abstract class WatchedNodeUpdaterWithReturnValueImpl implements WatchedNodeUpdaterWithReturnValue { 20 | 21 | private R returnValue; 22 | 23 | protected void setReturnValue(R returnValue) { 24 | this.returnValue = returnValue; 25 | } 26 | 27 | @Override 28 | public R getReturnValue() { 29 | return returnValue; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-core/src/main/java/com/liveramp/hank/zookeeper/WatchedString.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.zookeeper; 17 | 18 | import org.apache.zookeeper.KeeperException; 19 | 20 | public class WatchedString extends WatchedNode { 21 | 22 | public WatchedString(ZooKeeperPlus zk, String nodePath, boolean waitForCreation) throws KeeperException, InterruptedException { 23 | super(zk, nodePath, waitForCreation); 24 | } 25 | 26 | @Override 27 | protected String decode(byte[] data) { 28 | if (data == null) { 29 | return null; 30 | } 31 | return new String(data); 32 | } 33 | 34 | @Override 35 | protected byte[] encode(String v) { 36 | if (v == null) { 37 | return null; 38 | } 39 | return v.getBytes(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/coordinator/zk/MockHostCommandQueueChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.liveramp.hank.coordinator.zk; 5 | 6 | import com.liveramp.hank.coordinator.Host; 7 | import com.liveramp.hank.coordinator.HostCommandQueueChangeListener; 8 | 9 | import static org.junit.Assert.fail; 10 | 11 | final class MockHostCommandQueueChangeListener implements HostCommandQueueChangeListener { 12 | 13 | public Host calledWith; 14 | 15 | @Override 16 | public void onCommandQueueChange(Host hostConfig) { 17 | calledWith = hostConfig; 18 | synchronized (this) { 19 | notifyAll(); 20 | } 21 | } 22 | 23 | public void waitForNotification() throws Exception { 24 | waitForNotification(false); 25 | } 26 | 27 | public void waitForNotification(boolean timeoutOk) throws Exception { 28 | synchronized (this) { 29 | if (calledWith != null) { 30 | return; 31 | } 32 | long start = System.currentTimeMillis(); 33 | this.wait(15000); 34 | long end = System.currentTimeMillis(); 35 | if (calledWith != null) { 36 | return; 37 | } 38 | if (!timeoutOk && end - start > 15000) { 39 | fail("timed out waiting for notification!"); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/coordinator/zk/MockHostStateChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.liveramp.hank.coordinator.zk; 5 | 6 | import com.liveramp.hank.coordinator.HostState; 7 | import com.liveramp.hank.zookeeper.WatchedNodeListener; 8 | 9 | public final class MockHostStateChangeListener implements WatchedNodeListener { 10 | 11 | HostState calledWith; 12 | 13 | @Override 14 | public void onWatchedNodeChange(HostState state) { 15 | calledWith = state; 16 | synchronized (this) { 17 | notifyAll(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/coordinator/zk/TestZkPartitionProperties.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.coordinator.zk; 2 | 3 | import com.liveramp.hank.test.ZkTestCase; 4 | import com.liveramp.hank.zookeeper.ZkPath; 5 | import org.junit.Test; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | public class TestZkPartitionProperties extends ZkTestCase { 10 | @Test 11 | public void testCreate() throws Exception { 12 | ZkPartitionProperties pi = ZkPartitionProperties.create(getZk(), getRoot(), 1, 15000, 550); 13 | assertEquals(1, pi.getPartitionNumber()); 14 | assertEquals(15000, pi.getNumBytes()); 15 | assertEquals(550, pi.getNumRecords()); 16 | 17 | // should not throw an exception 18 | pi = ZkPartitionProperties.create(getZk(), getRoot(), 1, 15000, 550); 19 | assertEquals(1, pi.getPartitionNumber()); 20 | assertEquals(15000, pi.getNumBytes()); 21 | assertEquals(550, pi.getNumRecords()); 22 | } 23 | 24 | @Test 25 | public void testLoad() throws Exception { 26 | ZkPartitionProperties.create(getZk(), getRoot(), 1, 15000, 550); 27 | 28 | ZkPartitionProperties pi = new ZkPartitionProperties(getZk(), ZkPath.append(getRoot(), "part-1")); 29 | assertEquals(1, pi.getPartitionNumber()); 30 | assertEquals(15000, pi.getNumBytes()); 31 | assertEquals(550, pi.getNumRecords()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/hasher/TestLeftPaddedIdentityHasher.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.hasher; 2 | 3 | import org.junit.Test; 4 | 5 | import java.nio.ByteBuffer; 6 | import java.util.Arrays; 7 | 8 | import static org.junit.Assert.assertTrue; 9 | import static org.junit.Assert.fail; 10 | 11 | public class TestLeftPaddedIdentityHasher { 12 | 13 | @Test 14 | public void testPad() throws Exception { 15 | 16 | LeftPaddedIdentityHasher hasher = new LeftPaddedIdentityHasher(); 17 | 18 | byte[] result = new byte[4]; 19 | hasher.hash(ByteBuffer.wrap(new byte[]{1, 2}), 4, result); 20 | assertTrue(Arrays.equals(new byte[]{0,0,1,2}, result)); 21 | 22 | byte[] result2 = new byte[4]; 23 | hasher.hash(ByteBuffer.wrap(new byte[]{1, 2, 3, 4}), 4, result2); 24 | assertTrue(Arrays.equals(new byte[]{1, 2, 3, 4}, result2)); 25 | 26 | try{ 27 | byte[] result3 = new byte[4]; 28 | hasher.hash(ByteBuffer.wrap(new byte[]{1, 2, 3, 4, 5}), 4, result3); 29 | fail(); 30 | }catch(Exception e){ 31 | // cool 32 | } 33 | 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/storage/echo/TestEchoReader.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.echo; 2 | 3 | import com.liveramp.hank.test.BaseTestCase; 4 | import com.liveramp.hank.storage.ReaderResult; 5 | import org.junit.Test; 6 | 7 | import java.nio.ByteBuffer; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertTrue; 11 | 12 | public class TestEchoReader extends BaseTestCase { 13 | @Test 14 | public void testIt() throws Exception { 15 | EchoReader r = new EchoReader(57); 16 | ReaderResult result = new ReaderResult(); 17 | r.get(ByteBuffer.wrap(new byte[]{1, 2, 3}), result); 18 | assertTrue(result.isFound()); 19 | assertEquals("Original value: 01 02 03 Assigned to partition number: 57", new String(result.getBuffer().array())); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/zookeeper/TestWatchedLong.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.zookeeper; 2 | 3 | import com.liveramp.commons.test.WaitUntil; 4 | import com.liveramp.hank.test.ZkTestCase; 5 | import org.apache.zookeeper.CreateMode; 6 | import org.apache.zookeeper.ZooDefs.Ids; 7 | import org.junit.Test; 8 | 9 | import static org.junit.Assert.assertNull; 10 | import static org.junit.Assert.assertEquals; 11 | 12 | public class TestWatchedLong extends ZkTestCase { 13 | @Test 14 | public void testIt() throws Exception { 15 | final ZooKeeperPlus zk = getZk(); 16 | final String nodePath = ZkPath.append(getRoot(), "watchedNode"); 17 | zk.create(nodePath, "1".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); 18 | final WatchedLong wl = new WatchedLong(zk, nodePath); 19 | assertEquals(Long.valueOf(1), wl.get()); 20 | 21 | zk.setData(nodePath, "55".getBytes(), -1); 22 | WaitUntil.orDie(() -> Long.valueOf(55).equals(wl.get())); 23 | assertEquals(Long.valueOf(55), wl.get()); 24 | 25 | zk.setData(nodePath, null, -1); 26 | WaitUntil.orDie(() -> wl.get() == null); 27 | assertNull(wl.get()); 28 | 29 | final WatchedLong wl2 = new WatchedLong(zk, nodePath); 30 | WaitUntil.orDie(() -> null == wl2.get()); 31 | assertNull(wl2.get()); 32 | wl2.set(22L); 33 | WaitUntil.orDie(() -> Long.valueOf(22).equals(wl2.get()) && Long.valueOf(22).equals(wl.get())); 34 | assertEquals(Long.valueOf(22), wl2.get()); 35 | assertEquals(Long.valueOf(22), wl.get()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hank-core/src/test/java/com/liveramp/hank/zookeeper/TestZooKeeperPlus.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.zookeeper; 2 | 3 | import java.util.Arrays; 4 | 5 | import org.apache.zookeeper.CreateMode; 6 | import org.apache.zookeeper.KeeperException; 7 | import org.junit.Test; 8 | 9 | import com.liveramp.hank.test.ZkTestCase; 10 | 11 | import static org.junit.Assert.assertTrue; 12 | 13 | public class TestZooKeeperPlus extends ZkTestCase { 14 | 15 | @Test 16 | public void testIt() throws Exception { 17 | 18 | final ZooKeeperPlus zk = getZk(); 19 | 20 | zk.ensureCreated("/", null, CreateMode.PERSISTENT); 21 | assertExists("/", zk); 22 | 23 | zk.ensureCreated("/simple", "1".getBytes(), CreateMode.PERSISTENT); 24 | assertExists("/simple", zk); 25 | 26 | zk.ensureCreated("/simple", "2".getBytes(), CreateMode.PERSISTENT); 27 | assertExists("/simple", zk); 28 | assertTrue(Arrays.equals(zk.getData("/simple", false, null), "1".getBytes())); 29 | 30 | zk.ensureCreated("/deeper/file", null, CreateMode.PERSISTENT); 31 | assertExists("/deeper/file", zk); 32 | assertExists("/deeper", zk); 33 | 34 | zk.ensureCreated("/simple/even/deeper", "3".getBytes(), CreateMode.PERSISTENT); 35 | assertTrue(Arrays.equals(zk.getData("/simple", false, null), "1".getBytes())); 36 | 37 | } 38 | 39 | private void assertExists(String path, ZooKeeperPlus zkp) throws KeeperException, InterruptedException { 40 | assertTrue(zkp.exists(path, false) != null); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /hank-server/conf/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/hank-server/conf/.gitignore -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/cascading/FlowConnectorFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.cascading; 18 | 19 | import cascading.flow.FlowConnector; 20 | 21 | import java.util.Properties; 22 | 23 | public interface FlowConnectorFactory { 24 | public abstract FlowConnector create(Properties props); 25 | } 26 | 27 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/cascading/HadoopFlowConnectorFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.cascading; 18 | 19 | import cascading.flow.FlowConnector; 20 | import cascading.flow.hadoop.HadoopFlowConnector; 21 | 22 | import java.util.Properties; 23 | 24 | public class HadoopFlowConnectorFactory implements FlowConnectorFactory{ 25 | 26 | private final Properties properties; 27 | 28 | public HadoopFlowConnectorFactory(){ 29 | this(new Properties()); 30 | } 31 | 32 | public HadoopFlowConnectorFactory(Properties properties){ 33 | this.properties = properties; 34 | } 35 | 36 | @Override 37 | public FlowConnector create(Properties props) { 38 | Properties properties = new Properties(this.properties); 39 | properties.putAll(props); 40 | return new HadoopFlowConnector(properties); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/config/SimpleDataDirectoriesConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | import java.util.Collection; 20 | import java.util.Collections; 21 | import java.util.HashSet; 22 | import java.util.Set; 23 | 24 | public class SimpleDataDirectoriesConfigurator implements DataDirectoriesConfigurator { 25 | 26 | private final Set dataDirectories; 27 | 28 | public SimpleDataDirectoriesConfigurator(String dataDirectory) { 29 | this.dataDirectories = Collections.singleton(dataDirectory); 30 | } 31 | 32 | public SimpleDataDirectoriesConfigurator(Collection dataDirectories) { 33 | this.dataDirectories = new HashSet(); 34 | this.dataDirectories.addAll(dataDirectories); 35 | } 36 | 37 | @Override 38 | public Set getDataDirectories() { 39 | return dataDirectories; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/DomainBuilderMapperDefault.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hadoop; 18 | 19 | import org.apache.hadoop.io.BytesWritable; 20 | 21 | public class DomainBuilderMapperDefault extends DomainBuilderMapper { 22 | 23 | @Override 24 | protected KeyValuePair buildHankKeyValue(BytesWritable key, BytesWritable value) { 25 | return new KeyValuePair(key, value); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/DomainBuilderPartitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hadoop; 18 | 19 | import org.apache.hadoop.mapred.JobConf; 20 | import org.apache.hadoop.mapred.Partitioner; 21 | 22 | 23 | public class DomainBuilderPartitioner implements Partitioner { 24 | 25 | @Override 26 | public void configure(JobConf conf) { 27 | } 28 | 29 | @Override 30 | public int getPartition(KeyAndPartitionWritableComparable key, ValueWritable value, int numPartitions) { 31 | return key.getPartition() % numPartitions; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/DomainBuilderReducer.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.hadoop; 2 | 3 | import java.io.IOException; 4 | import java.util.Iterator; 5 | 6 | import org.apache.hadoop.mapred.JobConf; 7 | import org.apache.hadoop.mapred.OutputCollector; 8 | import org.apache.hadoop.mapred.Reducer; 9 | import org.apache.hadoop.mapred.Reporter; 10 | 11 | public class DomainBuilderReducer implements Reducer { 12 | 13 | @Override 14 | public void configure(JobConf conf) { 15 | } 16 | 17 | @Override 18 | public void close() throws IOException { 19 | } 20 | 21 | @Override 22 | public void reduce(KeyAndPartitionWritableComparable keyAndPartitionWritableComparable, 23 | Iterator iterator, 24 | OutputCollector outputCollector, 25 | Reporter reporter) throws IOException { 26 | while (iterator.hasNext()) { 27 | outputCollector.collect(keyAndPartitionWritableComparable.getKeyAndPartitionWritable(), iterator.next()); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/DomainVersionNumberAndNumPartitions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hadoop; 18 | 19 | public class DomainVersionNumberAndNumPartitions { 20 | 21 | private final Integer domainVersionNumber; 22 | private final Integer numPartitions; 23 | 24 | public DomainVersionNumberAndNumPartitions(Integer domainVersionNumber, Integer numPartitions) { 25 | this.domainVersionNumber = domainVersionNumber; 26 | this.numPartitions = numPartitions; 27 | } 28 | 29 | public Integer getDomainVersionNumber() { 30 | return domainVersionNumber; 31 | } 32 | 33 | public Integer getNumPartitions() { 34 | return numPartitions; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/KeyValuePair.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hadoop; 18 | 19 | import org.apache.hadoop.io.BytesWritable; 20 | 21 | public class KeyValuePair { 22 | 23 | BytesWritable key; 24 | BytesWritable value; 25 | 26 | public KeyValuePair(BytesWritable key, BytesWritable value) { 27 | this.key = key; 28 | this.value = value; 29 | } 30 | 31 | public KeyValuePair(byte[] key, byte[] value) { 32 | this.key = new BytesWritable(key); 33 | this.value = new BytesWritable(value); 34 | } 35 | 36 | public BytesWritable getKey() { 37 | return key; 38 | } 39 | 40 | public BytesWritable getValue() { 41 | return value; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return ""; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/hadoop/PartitionIntWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.hadoop; 18 | 19 | import org.apache.hadoop.io.IntWritable; 20 | 21 | public class PartitionIntWritable extends IntWritable { 22 | 23 | public PartitionIntWritable() { 24 | super(); 25 | } 26 | 27 | public PartitionIntWritable(int value) { 28 | super(value); 29 | } 30 | 31 | @Override 32 | public boolean equals(Object other) { 33 | return other != null 34 | && (PartitionIntWritable.class.equals(other.getClass())) 35 | && (get() == ((PartitionIntWritable) other).get()); 36 | } 37 | 38 | // Re-implementing hashCode in this way guarantees that the reduce partition computed for that object will be the same 39 | // as the Hank partition this object represents. 40 | @Override 41 | public int hashCode() { 42 | return get(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/partition_server/IUpdateManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.partition_server; 17 | 18 | import java.io.IOException; 19 | 20 | interface IUpdateManager { 21 | public void update() throws IOException; 22 | } 23 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/ring_group_conductor/LiveReplicaStatus.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ring_group_conductor; 2 | 3 | enum LiveReplicaStatus { 4 | UNDER_REPLICATED, 5 | REPLICATED, 6 | OVER_REPLICATED; 7 | 8 | public boolean isFullyReplicated() { 9 | return this == REPLICATED || this == OVER_REPLICATED; 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/ring_group_conductor/RingGroupTransitionFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.ring_group_conductor; 17 | 18 | import java.io.IOException; 19 | 20 | import com.liveramp.hank.coordinator.Coordinator; 21 | import com.liveramp.hank.coordinator.RingGroup; 22 | 23 | public interface RingGroupTransitionFunction { 24 | 25 | public void manageTransitions(Coordinator coordinator, RingGroup ringGroup) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /hank-server/src/main/java/com/liveramp/hank/storage/mock/MockDeleter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.mock; 18 | 19 | import com.liveramp.hank.storage.Deleter; 20 | 21 | import java.io.IOException; 22 | 23 | public class MockDeleter implements Deleter { 24 | 25 | private boolean hasDeleted = false; 26 | 27 | public MockDeleter(int partNum) { 28 | } 29 | 30 | @Override 31 | public void delete() throws IOException { 32 | hasDeleted = true; 33 | } 34 | 35 | public boolean hasDeleted() { 36 | return hasDeleted; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/fixtures/PartitionServerRunnable.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.fixtures; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.liveramp.hank.config.PartitionServerConfigurator; 7 | import com.liveramp.hank.partition_server.PartitionServer; 8 | 9 | public class PartitionServerRunnable implements Runnable { 10 | private static final Logger LOG = LoggerFactory.getLogger(PartitionServerRunnable.class); 11 | 12 | private PartitionServer server; 13 | private final PartitionServerConfigurator configurator; 14 | 15 | public PartitionServerRunnable(PartitionServerConfigurator configurator) throws Exception { 16 | this.configurator = configurator; 17 | } 18 | 19 | public void run() { 20 | try { 21 | server = new PartitionServer(configurator, "localhost"); 22 | server.run(); 23 | } catch (Throwable t) { 24 | LOG.error("crap, some exception...", t); 25 | } 26 | } 27 | 28 | public void pleaseStop() throws Exception { 29 | server.stopSynchronized(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/fixtures/RingGroupConductorRunnable.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.fixtures; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.liveramp.hank.config.RingGroupConductorConfigurator; 7 | import com.liveramp.hank.ring_group_conductor.RingGroupConductor; 8 | 9 | public class RingGroupConductorRunnable implements Runnable { 10 | private static final Logger LOG = LoggerFactory.getLogger(RingGroupConductorRunnable.class); 11 | 12 | private RingGroupConductorConfigurator configurator; 13 | private RingGroupConductor daemon; 14 | 15 | public RingGroupConductorRunnable(RingGroupConductorConfigurator configurator) throws Exception { 16 | this.configurator = configurator; 17 | } 18 | 19 | public void run() { 20 | try { 21 | daemon = new RingGroupConductor(configurator); 22 | daemon.run(); 23 | } catch (Exception e) { 24 | LOG.error("crap, some exception", e); 25 | } 26 | } 27 | 28 | public void pleaseStop() { 29 | daemon.stop(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/partition_server/MockPartitionUpdater.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.liveramp.hank.partition_server; 17 | 18 | import com.liveramp.hank.coordinator.DomainVersion; 19 | import com.liveramp.hank.storage.PartitionUpdater; 20 | 21 | import java.io.IOException; 22 | 23 | public class MockPartitionUpdater implements PartitionUpdater { 24 | 25 | private boolean updated = false; 26 | public Integer updatedToVersion = null; 27 | 28 | public void setUpdated(boolean updated) { 29 | this.updated = updated; 30 | } 31 | 32 | public boolean isUpdated() { 33 | return updated; 34 | } 35 | 36 | @Override 37 | public void updateTo(DomainVersion updatingToVersion, PartitionUpdateTaskStatistics statistics) throws IOException { 38 | updatedToVersion = updatingToVersion.getVersionNumber(); 39 | setUpdated(true); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/partition_server/MockUpdateManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.partition_server; 18 | 19 | import java.io.IOException; 20 | 21 | public class MockUpdateManager implements IUpdateManager { 22 | @Override 23 | public void update() throws IOException { 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/partition_server/TestDomainAccessor.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.partition_server; 2 | 3 | import com.liveramp.hank.test.BaseTestCase; 4 | import com.liveramp.hank.test.coordinator.MockHostDomain; 5 | import com.liveramp.hank.test.coordinator.MockHostDomainPartition; 6 | import com.liveramp.hank.coordinator.mock.MockDomain; 7 | import com.liveramp.hank.test.partitioner.MapPartitioner; 8 | import com.liveramp.hank.storage.ReaderResult; 9 | import com.liveramp.hank.storage.mock.MockReader; 10 | import org.junit.Test; 11 | 12 | import java.nio.ByteBuffer; 13 | 14 | public class TestDomainAccessor extends BaseTestCase { 15 | @Test 16 | public void testSetUpAndServe() throws Exception { 17 | PartitionAccessor partitionAccessors[] = new PartitionAccessor[1]; 18 | 19 | ByteBuffer key = ByteBuffer.wrap("key".getBytes()); 20 | ByteBuffer nullKey = ByteBuffer.wrap("nullKey".getBytes()); 21 | 22 | // setup DomainAccessor 23 | partitionAccessors[0] = new PartitionAccessor(new MockHostDomainPartition(0, 1), 24 | new MockReader(null, 1, "v".getBytes(), null)); 25 | // MapPartitioner maps both 'key' and 'nullkey' to partitionAccessors[0] 26 | DomainAccessor drs = new DomainAccessor(new MockHostDomain(new MockDomain("domain")), partitionAccessors, 27 | new MapPartitioner(key, 0, nullKey, 0), 0); 28 | 29 | drs.get(key, new ReaderResult()); 30 | drs.get(nullKey, new ReaderResult()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/storage/cueball/TestCueballDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.cueball; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | 6 | import java.io.File; 7 | 8 | import static org.junit.Assert.assertFalse; 9 | import static org.junit.Assert.assertTrue; 10 | 11 | public class TestCueballDeleter extends AbstractCueballTest { 12 | private final String LOCAL_ROOT = localTmpDir + "/local"; 13 | 14 | @Before 15 | public void setUp() throws Exception { 16 | new File(LOCAL_ROOT).mkdirs(); 17 | } 18 | 19 | @Test 20 | public void testDeleter() throws Exception { 21 | assertTrue(localFileExists(LOCAL_ROOT)); 22 | CueballDeleter deleter = new CueballDeleter(LOCAL_ROOT); 23 | deleter.delete(); 24 | assertFalse(localFileExists(LOCAL_ROOT)); 25 | } 26 | 27 | private boolean localFileExists(String localFile) { 28 | return new File(localFile).exists(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/storage/curly/MockCurlyMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.storage.curly; 18 | 19 | import com.liveramp.hank.storage.PartitionRemoteFileOps; 20 | 21 | import java.io.IOException; 22 | import java.util.List; 23 | 24 | public class MockCurlyMerger implements ICurlyMerger { 25 | 26 | public CurlyFilePath latestBase; 27 | public List deltaRemoteFiles; 28 | 29 | @Override 30 | public long[] merge(CurlyFilePath latestBase, 31 | List deltaRemoteFiles, 32 | PartitionRemoteFileOps partitionRemoteFileOps) throws IOException { 33 | this.latestBase = latestBase; 34 | this.deltaRemoteFiles = deltaRemoteFiles; 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hank-server/src/test/java/com/liveramp/hank/storage/curly/TestCurlyDeleter.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.storage.curly; 2 | 3 | import java.io.File; 4 | 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | import static org.junit.Assert.assertFalse; 9 | import static org.junit.Assert.assertTrue; 10 | 11 | public class TestCurlyDeleter extends AbstractCurlyTestBase { 12 | private final String LOCAL_ROOT = "/tmp/TestCurlyDeleter_local"; 13 | 14 | @Before 15 | public void setUp() throws Exception { 16 | super.setUp(); 17 | new File(LOCAL_ROOT).mkdirs(); 18 | } 19 | 20 | @Test 21 | public void testDeleter() throws Exception { 22 | assertTrue(localFileExists(LOCAL_ROOT)); 23 | CurlyDeleter deleter = new CurlyDeleter(LOCAL_ROOT); 24 | deleter.delete(); 25 | assertFalse(localFileExists(LOCAL_ROOT)); 26 | } 27 | 28 | private boolean localFileExists(String localFile) { 29 | return new File(localFile).exists(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-ui/bin/hank-ui: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BINDIR=`dirname "$0"` 4 | CONFDIR="$BINDIR/../conf" 5 | UI_JOBJAR=`find $BINDIR/../ -name 'hank-ui-*-jobjar.jar'` 6 | COMPONENT=$1 7 | shift 8 | COMMAND=$1 9 | shift 10 | ARGS=$@ 11 | 12 | # Load environment variables 13 | ENV_FILE="$CONFDIR"/hank-env.sh 14 | if [ -f $ENV_FILE ]; then 15 | . $ENV_FILE 16 | else 17 | echo "Warning: environment configuration file was not found: $ENV_FILE" 18 | fi 19 | 20 | UI_CLASS='com.liveramp.hank.ui.WebUiServer' 21 | 22 | # Default OPTS 23 | 24 | HANK_UI_OPTS="$HANK_UI_OPTS -XX:+UseConcMarkSweepGC" 25 | 26 | function start-ui { 27 | set-opts $HANK_UI_OPTS 28 | start-component $UI_JOBJAR $UI_CLASS $ARGS 29 | } 30 | 31 | function stop-ui { 32 | stop-component $UI_CLASS 33 | } 34 | 35 | function set-opts { 36 | JVM_OPTS=$@ 37 | } 38 | 39 | function start-component { 40 | JOBJAR=$1 41 | shift 42 | CLASSNAME=$1 43 | shift 44 | ARGS=$@ 45 | COMMAND="java $JVM_OPTS -cp $JOBJAR $CLASSNAME $ARGS" 46 | echo "$COMMAND" 47 | eval $COMMAND 48 | } 49 | 50 | function stop-component { 51 | CLASSNAME=$1 52 | for PID in `jps -ml | grep $CLASSNAME | cut -d ' ' -f 1`; do 53 | echo "Killing $PID $CLASSNAME" 54 | kill $PID 55 | done 56 | } 57 | 58 | function main { 59 | 60 | case $COMMAND in 61 | "start");; 62 | "stop");; 63 | "restart");; 64 | *) echo "Invalid command: $COMMAND"; exit 1;; 65 | esac 66 | 67 | case $COMPONENT in 68 | "ui");; 69 | *) echo "Invalid component: $COMPONENT" ; exit 1;; 70 | esac 71 | 72 | if [ $COMMAND == "restart" ]; then 73 | stop-$COMPONENT 74 | start-$COMPONENT 75 | else 76 | $COMMAND-$COMPONENT 77 | fi 78 | } 79 | 80 | main 81 | -------------------------------------------------------------------------------- /hank-ui/conf/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LiveRamp/hank/a1df2bcee94f58487547e56db7fb22c1c9cec897/hank-ui/conf/.gitignore -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/config/MockMonitorConfigurator.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.config; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | import com.liveramp.hank.coordinator.RingGroup; 8 | import com.liveramp.hank.monitor.notifier.Notifier; 9 | 10 | public class MockMonitorConfigurator implements MonitorConfigurator{ 11 | @Override 12 | public List getGlobalNotifiers() throws InvalidConfigurationException { 13 | return Lists.newArrayList(); 14 | } 15 | 16 | @Override 17 | public List getRingGroupNotifiers(RingGroup ringGroup) throws InvalidConfigurationException { 18 | return Lists.newArrayList(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/config/MonitorConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.config; 18 | 19 | import com.liveramp.hank.coordinator.RingGroup; 20 | import com.liveramp.hank.monitor.notifier.Notifier; 21 | 22 | import java.util.List; 23 | 24 | public interface MonitorConfigurator { 25 | 26 | public List getGlobalNotifiers() throws InvalidConfigurationException; 27 | 28 | public List getRingGroupNotifiers(RingGroup ringGroup) throws InvalidConfigurationException; 29 | } 30 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notification/AbstractNotification.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.monitor.notification; 2 | 3 | import java.util.Date; 4 | 5 | import org.joda.time.format.DateTimeFormat; 6 | import org.joda.time.format.DateTimeFormatter; 7 | 8 | import com.liveramp.hank.monitor.notifier.Notification; 9 | 10 | public abstract class AbstractNotification implements Notification { 11 | 12 | private final Date date; 13 | private static final String DATE_FORMAT = "yy/MM/dd HH:mm:ss"; 14 | private static final DateTimeFormatter dateFormat = DateTimeFormat.forPattern(DATE_FORMAT); 15 | 16 | public AbstractNotification() { 17 | date = new Date(); 18 | } 19 | 20 | @Override 21 | public Date getDate() { 22 | return date; 23 | } 24 | 25 | protected abstract String formatCore(NotificationFormatter formatter); 26 | 27 | @Override 28 | public final String format(NotificationFormatter formatter) { 29 | return dateFormat.print(date.getTime()) + " " + formatCore(formatter); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notification/EmailNotificationFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notification; 18 | 19 | public class EmailNotificationFormatter implements NotificationFormatter { 20 | 21 | private final String webUiUrl; 22 | 23 | public EmailNotificationFormatter(String webUiUrl) { 24 | this.webUiUrl = webUiUrl; 25 | } 26 | 27 | @Override 28 | public String getWebUiLink(String absolutePath, String text) { 29 | return "" + text + ""; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notification/NotificationFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notification; 18 | 19 | public interface NotificationFormatter { 20 | 21 | public String getWebUiLink(String absolutePath, String text); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notification/StringNotification.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notification; 18 | 19 | import com.liveramp.hank.monitor.notifier.Notification; 20 | 21 | public class StringNotification extends AbstractNotification implements Notification { 22 | 23 | private final String notification; 24 | 25 | public StringNotification(final String notification) { 26 | this.notification = notification; 27 | } 28 | 29 | @Override 30 | public String formatCore(NotificationFormatter formatter) { 31 | return notification; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/Notification.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notifier; 18 | 19 | import com.liveramp.hank.monitor.notification.NotificationFormatter; 20 | 21 | import java.util.Date; 22 | 23 | public interface Notification { 24 | 25 | public Date getDate(); 26 | 27 | public String format(NotificationFormatter formatter); 28 | } 29 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/Notifier.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notifier; 18 | 19 | public interface Notifier { 20 | 21 | public void doNotify(Notification notification); 22 | 23 | public void stop(); 24 | } 25 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/NotifierFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 LiveRamp 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.liveramp.hank.monitor.notifier; 18 | 19 | import com.liveramp.hank.config.InvalidConfigurationException; 20 | 21 | import java.util.Map; 22 | 23 | public interface NotifierFactory { 24 | 25 | public void validate(Map configuration) throws InvalidConfigurationException; 26 | 27 | public Notifier createNotifier(Map configuration, 28 | String name, 29 | String webUiUrl); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/db/ISQLNotifierConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.monitor.notifier.db; 2 | 3 | public interface ISQLNotifierConfiguration { 4 | String getDriver(); 5 | 6 | String getConnectionUrl(); 7 | 8 | String getUsername(); 9 | 10 | String getPassword(); 11 | 12 | String getTargetVersionTable(); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/db/SQLNotifier.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.monitor.notifier.db; 2 | 3 | import com.liveramp.hank.monitor.notifier.Notification; 4 | import com.liveramp.hank.monitor.notifier.Notifier; 5 | import org.slf4j.Logger; import org.slf4j.LoggerFactory; 6 | 7 | import java.sql.Connection; 8 | import java.sql.DriverManager; 9 | import java.sql.SQLException; 10 | 11 | public class SQLNotifier implements Notifier { 12 | private static Logger LOG = LoggerFactory.getLogger(SQLNotifier.class); 13 | 14 | private Connection connection; 15 | private ISQLNotifierConfiguration configuration; 16 | 17 | public SQLNotifier(ISQLNotifierConfiguration configuration) { 18 | this.configuration = configuration; 19 | } 20 | 21 | @Override 22 | public void doNotify(Notification notification) { 23 | } 24 | 25 | @Override 26 | public void stop() { 27 | if (connection != null) { 28 | try { 29 | connection.close(); 30 | } catch (SQLException e) { 31 | LOG.warn("Cannot close database", e); 32 | } 33 | } 34 | } 35 | 36 | private Connection getConnection() { 37 | if (connection == null) { 38 | try { 39 | Class.forName(configuration.getDriver()); 40 | connection = DriverManager.getConnection(configuration.getConnectionUrl(), configuration.getUsername(), configuration.getPassword()); 41 | } catch (ClassNotFoundException e) { 42 | LOG.warn("Cannot find SQL driver " + configuration.getDriver(), e); 43 | } catch (SQLException e) { 44 | LOG.warn("Cannot get SQL connection", e); 45 | } 46 | } 47 | return connection; 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/db/SQLNotifierConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.monitor.notifier.db; 2 | 3 | import org.slf4j.Logger; import org.slf4j.LoggerFactory; 4 | 5 | public class SQLNotifierConfiguration implements ISQLNotifierConfiguration { 6 | private static Logger LOG = LoggerFactory.getLogger(SQLNotifierConfiguration.class); 7 | 8 | private String driver; 9 | private String connectionUrl; 10 | private String username; 11 | private String password; 12 | private String targetVersionTable; 13 | 14 | public SQLNotifierConfiguration(String driver, String connectionUrl, String username, String password, String targetVersionTable) { 15 | this.driver = driver; 16 | this.connectionUrl = connectionUrl; 17 | this.username = username; 18 | this.password = password; 19 | this.targetVersionTable = targetVersionTable; 20 | } 21 | 22 | @Override 23 | public String getDriver() { 24 | return driver; 25 | } 26 | 27 | @Override 28 | public String getConnectionUrl() { 29 | return connectionUrl; 30 | } 31 | 32 | @Override 33 | public String getUsername() { 34 | return username; 35 | } 36 | 37 | @Override 38 | public String getPassword() { 39 | return password; 40 | } 41 | 42 | @Override 43 | public String getTargetVersionTable() { 44 | return targetVersionTable; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/monitor/notifier/mock/MockNotifierFactory.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.monitor.notifier.mock; 2 | 3 | import com.liveramp.hank.config.InvalidConfigurationException; 4 | import com.liveramp.hank.monitor.notifier.Notifier; 5 | import com.liveramp.hank.monitor.notifier.NotifierFactory; 6 | 7 | import java.util.Map; 8 | 9 | public class MockNotifierFactory implements NotifierFactory { 10 | 11 | @Override 12 | public void validate(Map configuration) throws InvalidConfigurationException { 13 | 14 | } 15 | 16 | @Override 17 | public Notifier createNotifier(Map configuration, 18 | String name, 19 | String webUiUrl) { 20 | return new MockNotifier(configuration); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/ClientCache.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ui; 2 | 3 | import com.liveramp.hank.client.HankSmartClient; 4 | import com.liveramp.hank.coordinator.Coordinator; 5 | import com.liveramp.hank.coordinator.RingGroup; 6 | import com.liveramp.hank.generated.SmartClient; 7 | import com.liveramp.hank.generated.SmartClient.Iface; 8 | import org.apache.thrift.TException; 9 | 10 | import java.io.IOException; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | 14 | public final class ClientCache implements IClientCache { 15 | private static final Map cachedClients = new HashMap(); 16 | private final Coordinator coordinator; 17 | 18 | public ClientCache(Coordinator coordinator) { 19 | this.coordinator = coordinator; 20 | } 21 | 22 | public synchronized SmartClient.Iface getSmartClient(RingGroup ringGroup) throws IOException, TException { 23 | Iface client = cachedClients.get(ringGroup.getName()); 24 | if (client == null) { 25 | client = new HankSmartClient(coordinator, ringGroup.getName()); 26 | cachedClients.put(ringGroup.getName(), client); 27 | } 28 | return client; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/ClientMetadataComparator.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ui; 2 | 3 | import com.liveramp.hank.generated.ClientMetadata; 4 | 5 | import java.util.Comparator; 6 | 7 | public class ClientMetadataComparator implements Comparator { 8 | @Override 9 | public int compare(ClientMetadata a, ClientMetadata b) { 10 | return a.get_host().compareTo(b.get_host()); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/IClientCache.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ui; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.thrift.TException; 6 | 7 | import com.liveramp.hank.coordinator.RingGroup; 8 | import com.liveramp.hank.generated.SmartClient; 9 | 10 | public interface IClientCache { 11 | 12 | public abstract SmartClient.Iface getSmartClient(RingGroup rgc) throws IOException, TException; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/URLEnc.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ui; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | 5 | public final class URLEnc { 6 | private URLEnc() { 7 | } 8 | 9 | public static String encode(String s) { 10 | try { 11 | return java.net.URLEncoder.encode(s, "UTF-8"); 12 | } catch (UnsupportedEncodingException e) { 13 | throw new RuntimeException(e); 14 | } 15 | } 16 | 17 | public static String decode(String s) { 18 | try { 19 | return java.net.URLDecoder.decode(s, "UTF-8"); 20 | } catch (UnsupportedEncodingException e) { 21 | throw new RuntimeException(e); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/_footer.jsp: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/_top_nav.jsp: -------------------------------------------------------------------------------- 1 | <%@ page import="com.liveramp.hank.coordinator.Coordinator" %> 2 | 3 | <% 4 | Coordinator coord = (Coordinator)getServletContext().getAttribute("coordinator"); 5 | %> 6 |
7 |
Hank
 8 |
Domains
 9 |
Domain Groups
 10 |
Ring Groups
 11 | 12 |
13 | <%--technically this could be any implementation, but it's going to be more confusing to the end 14 | user to say "Coordinator". can change this when there is a different production Coordinator impl--%> 15 | ZooKeeper 16 | <%=coord.getDataState()%> 17 |
18 | 19 |
20 | 21 | 22 |
23 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/controllers/Action.java: -------------------------------------------------------------------------------- 1 | package com.liveramp.hank.ui.controllers; 2 | 3 | import java.io.IOException; 4 | 5 | import javax.servlet.ServletException; 6 | import javax.servlet.http.HttpServletRequest; 7 | import javax.servlet.http.HttpServletResponse; 8 | 9 | public abstract class Action { 10 | 11 | protected abstract void action(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException; 12 | 13 | protected void redirect(String uri, HttpServletResponse resp) throws IOException { 14 | resp.sendRedirect(uri); 15 | resp.setStatus(HttpServletResponse.SC_OK); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /hank-ui/src/main/java/com/liveramp/hank/ui/index.jsp: -------------------------------------------------------------------------------- 1 | <%@ page language="java" contentType="text/html; charset=ISO-8859-1" 2 | pageEncoding="ISO-8859-1" %> 3 | 4 | <%@ page import="com.liveramp.hank.coordinator.*" %> 5 | <%@ page import="com.liveramp.hank.Hank" %> 6 | <% 7 | Coordinator coord = (Coordinator)getServletContext().getAttribute("coordinator"); 8 | %> 9 | 10 | 11 | 12 | 13 | Hank: Home 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |

Hank

22 | 23 |

System Summary

24 | 25 | <%= coord.getDomains().size() %> domains, 26 | <%= coord.getDomainGroups().size() %> domain groups, 27 | and <%= coord.getRingGroups().size() %> ring groups. 28 | 29 |

Coordinator

30 |
31 | <%= coord %> 32 |
33 | 34 |

Administration

35 |
Open administration panel
 36 | 37 |

Version Information

38 |
39 | Hank version <%= Hank.getVersion() %>, commit <%= Hank.getGitCommit() %> 40 |
41 |
42 | Please report bugs on GitHub issues page. 43 |
44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/assembly/dist.xml: -------------------------------------------------------------------------------- 1 | 4 | dist 5 | 6 | tar.gz 7 | 8 | 9 | 10 | ${project.basedir} 11 | / 12 | 13 | README* 14 | bin/ 15 | conf/ 16 | 17 | 18 | 19 | ${project.build.directory} 20 | / 21 | 22 | ${project.name}-${project.version}-jobjar.jar 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/assembly/jobjar.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | jobjar 6 | 7 | jar 8 | 9 | false 10 | 11 | 12 | / 13 | true 14 | true 15 | runtime 16 | 17 | 18 | 19 | --------------------------------------------------------------------------------